Merge pull request #164 from ate47/fix_windows

Fix non closed files

.github/workflows/mvn-windows.yaml

@@ -0,0 +1,21 @@
+name: Java CI (Windows)
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 15
+        uses: actions/setup-java@v1
+        with:
+          java-version: 15
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-maven-
+      - name: Run tests
+        run:  mvn test

hdt-api/src/main/java/org/rdfhdt/hdt/hdt/HDTManager.java

@@ -254,9 +254,10 @@ public static HDT loadIndexedHDT(InputStream hdtFileName, ProgressListener liste
 	 * Return an indexed HDT that is efficient for all kind of queries, given a not indexed HDT.
 	 * @param hdt file path to index
 	 * @param listener Listener to get notified of loading progress. Can be null if no notifications needed.
+	 * @throws IOException when the file cannot be found
 	 * @return HDT
 	 */
-	public static HDT indexedHDT(HDT hdt, ProgressListener listener) {
+	public static HDT indexedHDT(HDT hdt, ProgressListener listener) throws IOException {
 		return HDTManager.getInstance().doIndexedHDT(hdt, listener);
 	}
 
@@ -346,7 +347,7 @@ public static HDT diffHDTBit(String location, String hdtFileName, Bitmap deleteB
 	protected abstract HDT doLoadIndexedHDT(String hdtFileName, ProgressListener listener, HDTOptions spec) throws IOException;
 	protected abstract HDT doLoadIndexedHDT(InputStream hdtFileName, ProgressListener listener, HDTOptions spec) throws IOException;
 	protected abstract HDT doMapIndexedHDT(String hdtFileName, ProgressListener listener, HDTOptions spec) throws IOException;
-	protected abstract HDT doIndexedHDT(HDT hdt, ProgressListener listener);
+	protected abstract HDT doIndexedHDT(HDT hdt, ProgressListener listener) throws IOException;
 	protected abstract HDT doGenerateHDT(String rdfFileName, String baseURI, RDFNotation rdfNotation, HDTOptions hdtFormat, ProgressListener listener) throws IOException, ParserException;
 	protected abstract HDT doGenerateHDT(Iterator<TripleString> iterator, String baseURI,	HDTOptions hdtFormat, ProgressListener listener) throws IOException;
 	protected abstract TripleWriter doGetHDTWriter(OutputStream out, String baseURI, HDTOptions hdtFormat) throws IOException;

hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/bitmap/Bitmap64Disk.java

@@ -29,14 +29,15 @@
 import org.rdfhdt.hdt.util.disk.LongArrayDisk;
 import org.rdfhdt.hdt.util.io.IOUtil;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
 /**
  * Version of Bitmap64 which is backed up on disk
  */
-public class Bitmap64Disk {
+public class Bitmap64Disk implements Closeable {
 
     // Constants
     protected final static int LOGW = 6;
@@ -205,4 +206,9 @@ public String toString() {
     public long getRealSizeBytes() {
         return words.length()*8;
     }
+
+    @Override
+    public void close() throws IOException {
+        words.close();
+    }
 }

hdt-java-core/src/main/java/org/rdfhdt/hdt/compact/sequence/SequenceLog64Map.java

@@ -52,6 +52,7 @@
 import org.rdfhdt.hdt.util.crc.CRC8;
 import org.rdfhdt.hdt.util.crc.CRCInputStream;
 import org.rdfhdt.hdt.util.crc.CRCOutputStream;
+import org.rdfhdt.hdt.util.io.CloseMappedByteBuffer;
 import org.rdfhdt.hdt.util.io.CountInputStream;
 import org.rdfhdt.hdt.util.io.IOUtil;
 
@@ -62,7 +63,7 @@
 public class SequenceLog64Map implements Sequence,Closeable { 
 	private static final byte W = 64;
 	private static final long LONGS_PER_BUFFER=128*1024*1024; // 128*8 = 1Gb per chunk.
-	private ByteBuffer [] buffers;
+	private CloseMappedByteBuffer[] buffers;
 	private FileChannel ch;
 	private int numbits;
 	private long numentries;
@@ -130,13 +131,16 @@ private void mapFiles(File f, long base) throws IOException {
 		long maxSize = base+SequenceLog64.numBytesFor(numbits, numentries);
 		int buffer = 0;
 		long block=0;
-		buffers = new ByteBuffer[ (int)(1L+numwords/LONGS_PER_BUFFER) ];
+		if (buffers != null) {
+			IOUtil.closeAll(buffers);
+		}
+		buffers = new CloseMappedByteBuffer[ (int)(1L+numwords/LONGS_PER_BUFFER) ];
 		while(block<numwords) {
 			long current = base+ buffer*8L*LONGS_PER_BUFFER;
 			long next = current+8L*LONGS_PER_BUFFER;
 			long length = Math.min(maxSize, next)-current;
 //			System.out.println("Ini: "+current+ " Max: "+ next+ " Length: "+length);
-			buffers[buffer] = ch.map(MapMode.READ_ONLY, current , length );
+			buffers[buffer] = IOUtil.mapChannel(f.getAbsolutePath(), ch, MapMode.READ_ONLY, current , length);
 			buffers[buffer].order(ByteOrder.LITTLE_ENDIAN);
 
 			block+=LONGS_PER_BUFFER;
@@ -176,8 +180,7 @@ private long getWord(long w) {
 			return lastword;
 		}
 
-		ByteBuffer buffer = buffers[(int)(w/LONGS_PER_BUFFER)];
-		return buffer.getLong((int)((w%LONGS_PER_BUFFER)*8));
+		return buffers[(int)(w/LONGS_PER_BUFFER)].getLong((int)((w%LONGS_PER_BUFFER)*8));
 	}
 
 	/* (non-Javadoc)
@@ -273,6 +276,7 @@ public void load(InputStream input, ProgressListener listener)
 
 	@Override
 	public void close() throws IOException {
+		IOUtil.closeAll(buffers);
 		buffers=null;
 		ch.close();
 	}

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/DictionaryCat.java

@@ -4,10 +4,11 @@
 import org.rdfhdt.hdt.dictionary.impl.utilCat.CatMappingBack;
 import org.rdfhdt.hdt.listener.ProgressListener;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.HashMap;
 
-public interface DictionaryCat {
+public interface DictionaryCat extends Closeable {
     void cat(Dictionary dictionary1, Dictionary dictionary2, ProgressListener listener) throws IOException;
     CatMappingBack getMappingS();
     long getNumShared();

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/DictionaryDiff.java

@@ -4,11 +4,12 @@
 import org.rdfhdt.hdt.dictionary.impl.utilCat.CatMapping;
 import org.rdfhdt.hdt.listener.ProgressListener;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
-public interface DictionaryDiff {
+public interface DictionaryDiff extends Closeable {
     /**
      * compute the diff of the previous dictionary
      * @param dictionary previous dictionary

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/impl/BaseDictionary.java

@@ -222,30 +222,15 @@ public CharSequence idToString(long id, TripleComponentRole role) {
 	}
 	@Override
 	public String dataTypeOfId(long id) {
-		try {
-			throw new IllegalAccessException("Method is not applicable on this dictionary");
-		} catch (IllegalAccessException e) {
-			e.printStackTrace();
-		}
-		return "";
+		throw new IllegalArgumentException("Method is not applicable on this dictionary");
 	}
 	@Override
 	public TreeMap<String, DictionarySection> getAllObjects() {
-		try {
-			throw new IllegalAccessException("Method is not applicable on this dictionary");
-		} catch (IllegalAccessException e) {
-			e.printStackTrace();
-		}
-		return null;
+		throw new IllegalArgumentException("Method is not applicable on this dictionary");
 	}
 	@Override
 	public long getNAllObjects() {
-		try {
-			throw new IllegalAccessException("Method is not applicable on this dictionary");
-		} catch (IllegalAccessException e) {
-			e.printStackTrace();
-		}
-		return 0;
+		throw new IllegalArgumentException("Method is not applicable on this dictionary");
 	}
 
 }

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/impl/FourSectionDictionary.java

@@ -45,6 +45,7 @@
 import org.rdfhdt.hdt.options.ControlInformation;
 import org.rdfhdt.hdt.options.HDTOptions;
 import org.rdfhdt.hdt.util.io.CountInputStream;
+import org.rdfhdt.hdt.util.io.IOUtil;
 import org.rdfhdt.hdt.util.listener.IntermediateListener;
 
 
@@ -164,9 +165,11 @@ public String getType() {
 
 	@Override
 	public void close() throws IOException {
-		shared.close();
-		subjects.close();
-		predicates.close();
-		objects.close();
+		IOUtil.closeAll(
+				shared,
+				subjects,
+				predicates,
+				objects
+		);
 	}
 }

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/impl/FourSectionDictionaryCat.java

@@ -28,9 +28,11 @@
 import org.rdfhdt.hdt.listener.ProgressListener;
 import org.rdfhdt.hdt.options.ControlInfo;
 import org.rdfhdt.hdt.options.ControlInformation;
+import org.rdfhdt.hdt.util.io.IOUtil;
 
 import java.io.*;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
 
@@ -191,33 +193,18 @@ public void cat(Dictionary dictionary1, Dictionary dictionary2, ProgressListener
         ci.setFormat(HDTVocabulary.DICTIONARY_TYPE_FOUR_SECTION);
         ci.setInt("elements", numSubjects+numPredicates+numObject+numShared);
 
-        try {
-            ci.save(new FileOutputStream(location + "dictionary"));
-            FileOutputStream outFinal = new FileOutputStream(location + "dictionary",true);
-            byte[] buf = new byte[100000];
+        try (FileOutputStream outFinal = new FileOutputStream(location + "dictionary")) {
+            ci.save(outFinal);
             for (int i = 1; i <= 4; i++) {
                 int j = i;
-                if (i == 4){
+                if (i == 4) {
                     j = 3;
-                } else if (j == 3){
+                } else if (j == 3) {
                     j = 4;
                 }
-                try {
-                    InputStream in = new FileInputStream(location + "section" + j);
-                    int b;
-                    while ((b = in.read(buf)) >= 0) {
-                        outFinal.write(buf, 0, b);
-                        outFinal.flush();
-                    }
-                    in.close();
-                    Files.delete(Paths.get(location + "section" + j));
-                } catch (FileNotFoundException e ){
-                    e.printStackTrace();
-                }
+                Files.copy(Path.of(location + "section" + j), outFinal);
+                Files.delete(Paths.get(location + "section" + j));
             }
-            outFinal.close();
-        } catch (IOException e) {
-            e.printStackTrace();
         }
         //calculate the inverse mapping for the subjects, i.e. from the new dictionary subject section to the old ones
         mappingS = new CatMappingBack(location,numSubjects+numShared);
@@ -246,23 +233,16 @@ public void cat(Dictionary dictionary1, Dictionary dictionary2, ProgressListener
             }
         }
     }
-    public void closeAll() {
+    @Override
+    public void close() throws IOException {
+        // iterate over all mappings and close them
         try {
-            // iterate over all mappings and close them
-            for(CatMapping mapping:allMappings.values()){
-                doClose(mapping);
-            }
-            doClose(mappingS);
-        } catch (IOException e) {
-            e.printStackTrace();
+            IOUtil.closeAll(allMappings.values());
+        } finally {
+            IOUtil.closeAll(mappingS);
         }
     }
 
-    private static void doClose(Closeable closeable) throws IOException {
-        if (closeable != null){
-            closeable.close();
-        }
-    }
     public CatMappingBack getMappingS() {
         return mappingS;
     }

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/impl/FourSectionDictionaryDiff.java

@@ -13,13 +13,15 @@
 import org.rdfhdt.hdt.listener.ProgressListener;
 import org.rdfhdt.hdt.options.ControlInfo;
 import org.rdfhdt.hdt.options.ControlInformation;
+import org.rdfhdt.hdt.util.io.IOUtil;
 
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -39,6 +41,15 @@ public FourSectionDictionaryDiff(String location) {
         this.location = location;
     }
 
+    @Override
+    public void close() throws IOException {
+        // iterate over all mappings and close them
+        try {
+            IOUtil.closeAll(allMappings.values());
+        } finally {
+            IOUtil.closeAll(mappingBack);
+        }
+    }
     @Override
     public void diff(Dictionary dictionary, Map<String, ModifiableBitmap> bitmaps, ProgressListener listener) throws IOException {
         allMappings.put("predicate", new CatMapping(location, "predicate", dictionary.getPredicates().getNumberOfElements()));
@@ -138,25 +149,16 @@ public void diff(Dictionary dictionary, Map<String, ModifiableBitmap> bitmaps, P
         try (OutputStream outFinal = new FileOutputStream(location + "dictionary")) {
             ci.save(outFinal);
 
-            byte[] buf = new byte[100000];
             for (int i = 1; i <= 4; i++) {
                 int j = i;
                 if (i == 4) {
                     j = 3;
                 } else if (j == 3) {
                     j = 4;
                 }
-                try (InputStream in = new FileInputStream(location + "section" + j)) {
-                    int b;
-                    while ((b = in.read(buf)) >= 0) {
-                        outFinal.write(buf, 0, b);
-                        outFinal.flush();
-                    }
-                }
+                Files.copy(Path.of(location + "section" + j), outFinal);
                 Files.delete(Paths.get(location + "section" + j));
             }
-        } catch (IOException e) {
-            e.printStackTrace();
         }
         mappingBack = new CatMapping(location, "back", numSubj + numShared);
 

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/impl/MultipleBaseDictionary.java

@@ -7,6 +7,7 @@
 import org.rdfhdt.hdt.dictionary.impl.section.PFCOptimizedExtractor;
 import org.rdfhdt.hdt.enums.DictionarySectionRole;
 import org.rdfhdt.hdt.enums.TripleComponentRole;
+import org.rdfhdt.hdt.exceptions.NotImplementedException;
 import org.rdfhdt.hdt.options.HDTOptions;
 import org.rdfhdt.hdt.util.LiteralsUtils;
 import org.rdfhdt.hdt.util.string.CompactString;
@@ -212,7 +213,7 @@ public TreeMap<String, DictionarySection> getAllObjects() {
 
     @Override
     public DictionarySection getObjects() {
-        return null;
+        throw new NotImplementedException();
     }
 
     @Override

hdt-java-core/src/main/java/org/rdfhdt/hdt/dictionary/impl/MultipleSectionDictionaryBig.java

@@ -34,7 +34,7 @@ public MultipleSectionDictionaryBig(HDTOptions spec) {
         // FIXME: Read type from spec.
         subjects = new PFCDictionarySectionBig(spec);
         predicates = new PFCDictionarySectionBig(spec);
-        objects = new TreeMap<String, DictionarySectionPrivate>();
+        objects = new TreeMap<>();
         shared = new PFCDictionarySectionBig(spec);
     }
 
@@ -46,7 +46,7 @@ public void load(TempDictionary other, ProgressListener listener) {
         IntermediateListener iListener = new IntermediateListener(listener);
         subjects.load(other.getSubjects(), iListener);
         predicates.load(other.getPredicates(), iListener);
-        Iterator iter = other.getObjects().getEntries();
+        Iterator<? extends CharSequence> iter = other.getObjects().getEntries();
 
         HashMap<String,Long> literalsCounts = ((HashDictionarySection)other.getObjects()).getLiteralsCounts();
         if(literalsCounts.containsKey("NO_DATATYPE"))