Clean up code; add better documentation for how to use other midi files; download zipped midi files and extract melody strings to a temporary file, if that file doesn't already exist

Author: DonaldAlan

dl4j-examples/src/main/java/org/deeplearning4j/examples/wip/advanced/modelling/melodl4j/MelodyModelingExample.java

@@ -31,34 +31,45 @@
 import org.deeplearning4j.nn.weights.WeightInit;
 import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
 import org.deeplearning4j.util.ModelSerializer;
+import org.nd4j.common.util.ArchiveUtils;
 import org.nd4j.linalg.activations.Activation;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.dataset.DataSet;
 import org.nd4j.linalg.factory.Nd4j;
 import org.nd4j.linalg.learning.config.RmsProp;
 import org.nd4j.linalg.lossfunctions.LossFunctions;
 
+import javax.sound.midi.InvalidMidiDataException;
 import java.io.*;
 import java.net.URL;
 import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 
 /**
  * LSTM  Symbolic melody modelling example, to compose music from symbolic melodies extracted from MIDI.
- * Based closely on LSTMCharModellingExample.java.
+ * LSTM logic is based closely on LSTMCharModellingExample.java.
  * See the README file in this directory for documentation.
  *
  * @author Alex Black, Donald A. Smith.
  */
 public class MelodyModelingExample {
-    final static String inputSymbolicMelodiesFilename = "bach-melodies-input.txt";
-    // Examples:  bach-melodies-input.txt, beatles-melodies-input.txt ,  pop-melodies-input.txt (large)
+    // If you want to change the MIDI files used in learning, create a zip file containing your MIDI
+    // files and replace the following path.  For example, you might use something like:
+    //final static String midiFileZipFileUrlPath = "file:d:/music/midi/classical-midi.zip";
+    final static String midiFileZipFileUrlPath = "http://waliberals.org/truthsite/music/bach-midi.zip";
 
-    final static String tmpDir = System.getProperty("java.io.tmpdir");
+    // For example "bach-midi.txt"
+    final static String inputSymbolicMelodiesFilename = getMelodiesFileNameFromURLPath(midiFileZipFileUrlPath);
 
-    final static String symbolicMelodiesInputFilePath = tmpDir + "/" + inputSymbolicMelodiesFilename;  // Point to melodies created by MidiMelodyExtractor.java
+    // Examples:  bach-melodies-input.txt, beatles-melodies-input.txt ,  pop-melodies-input.txt (large)
+    final static String tmpDir = System.getProperty("java.io.tmpdir");
+    final static String inputSymbolicMelodiesFilePath = tmpDir + "/" + inputSymbolicMelodiesFilename;  // Point to melodies created by MidiMelodyExtractor.java
     final static String composedMelodiesOutputFilePath = tmpDir + "/composition.txt"; // You can listen to these melodies by running PlayMelodyStrings.java against this file.
 
     //final static String symbolicMelodiesInputFilePath = "D:/tmp/bach-melodies.txt";
@@ -73,6 +84,8 @@ public static void main(String[] args) throws Exception {
             generationInitialization = args[1];
         }
 
+        makeMidiStringFileIfNecessary();
+
         int lstmLayerSize = 200;                    //Number of units in each LSTM layer
         int miniBatchSize = 32;                     //Size of mini batch to use when training
         int exampleLength = 500; //1000; 		    //Length of each training example sequence to use.
@@ -199,36 +212,78 @@ private static void learn(int miniBatchSize, int exampleLength, int numEpochs, i
         System.exit(0);
     }
 
-    public static void makeSureFileIsInTmpDir(String filename) {
+    public static File makeSureFileIsInTmpDir(String urlString) throws IOException {
+        final URL url = new URL(urlString);
+        final String filename = urlString.substring(1+urlString.lastIndexOf("/"));
         final File f = new File(tmpDir + "/" + filename);
-        if (!f.exists()) {
-            URL url = null;
-            try {
-                url = new URL("http://truthsite.org/music/" + filename);
-                FileUtils.copyURLToFile(url, f);
-            } catch (Exception exc) {
-                System.err.println("Error copying " + url + " to " + f);
-                throw new RuntimeException(exc);
-            }
+        if (f.exists()) {
+            System.out.println("Using existing " + f.getAbsolutePath());
+        } else {
+            FileUtils.copyURLToFile(url, f);
             if (!f.exists()) {
                 throw new RuntimeException(f.getAbsolutePath() + " does not exist");
             }
             System.out.println("File downloaded to " + f.getAbsolutePath());
-        } else {
-            System.out.println("Using existing text file at " + f.getAbsolutePath());
         }
+        return f;
     }
 
+    //https://stackoverflow.com/questions/10633595/java-zip-how-to-unzip-folder
+    public static void unzip(File zipFile, File targetDirFile) throws IOException {
+        InputStream is =  new FileInputStream(zipFile);
+        Path targetDir = targetDirFile.toPath();
+        targetDir = targetDir.toAbsolutePath();
+        try (ZipInputStream zipIn = new ZipInputStream(is)) {
+            for (ZipEntry ze; (ze = zipIn.getNextEntry()) != null; ) {
+                Path resolvedPath = targetDir.resolve(ze.getName()).normalize();
+                if (!resolvedPath.startsWith(targetDir)) {
+                    // see: https://snyk.io/research/zip-slip-vulnerability
+                    throw new RuntimeException("Entry with an illegal path: "
+                            + ze.getName());
+                }
+                if (ze.isDirectory()) {
+                    Files.createDirectories(resolvedPath);
+                } else {
+                    Files.createDirectories(resolvedPath.getParent());
+                    Files.copy(zipIn, resolvedPath);
+                }
+            }
+        }
+        is.close();
+    }
+    private static void makeMidiStringFileIfNecessary() throws IOException, InvalidMidiDataException {
+        final File inputMelodiesFile = new File(inputSymbolicMelodiesFilePath);
+        if (inputMelodiesFile.exists() && inputMelodiesFile.length()>1000) {
+            System.out.println("Using existing " + inputSymbolicMelodiesFilePath);
+            return;
+        }
+        final File midiZipFile = makeSureFileIsInTmpDir(midiFileZipFileUrlPath);
+        final String midiZipFileName = midiZipFile.getName();
+        final String midiZipFileNameWithoutSuffix = midiZipFileName.substring(0,midiZipFileName.lastIndexOf("."));
+        final File outputDirectoryFile  = new File(tmpDir,midiZipFileNameWithoutSuffix);
+        final String outputDirectoryPath = outputDirectoryFile.getAbsolutePath();
+        if (!outputDirectoryFile.exists()) {
+            outputDirectoryFile.mkdir();
+        }
+        if (!outputDirectoryFile.exists() || !outputDirectoryFile.isDirectory()) {
+            throw new IllegalStateException(outputDirectoryFile + " is not a directory or can't be created");
+        }
+        final PrintStream printStream = new PrintStream(inputSymbolicMelodiesFilePath);
+        System.out.println("Unzipping "+ midiZipFile.getAbsolutePath() + " to " + outputDirectoryPath);
+        unzip(midiZipFile, outputDirectoryFile);
+        System.out.println("Extracted " + midiZipFile.getAbsolutePath() + " to " + outputDirectoryPath);
+        MidiMelodyExtractor.processDirectoryAndWriteMelodyFile(outputDirectoryFile,inputMelodiesFile);
+        printStream.close();
+    }
     /**
      * Sets up and return a simple DataSetIterator that does vectorization based on the melody sample.
      *
      * @param miniBatchSize  Number of text segments in each training mini-batch
      * @param sequenceLength Number of characters in each text segment.
      */
     public static CharacterIterator getMidiIterator(int miniBatchSize, int sequenceLength) throws Exception {
-        makeSureFileIsInTmpDir(inputSymbolicMelodiesFilename);
         final char[] validCharacters = MelodyStrings.allValidCharacters.toCharArray(); //Which characters are allowed? Others will be removed
-        return new CharacterIterator(symbolicMelodiesInputFilePath, Charset.forName("UTF-8"),
+        return new CharacterIterator(inputSymbolicMelodiesFilePath, Charset.forName("UTF-8"),
             miniBatchSize, sequenceLength, validCharacters, new Random(12345), MelodyStrings.COMMENT_STRING);
     }
 
@@ -312,5 +367,13 @@ public static int sampleFromDistribution(double[] distribution, Random rng) {
         //Should be extremely unlikely to happen if distribution is a valid probability distribution
         throw new IllegalArgumentException("Distribution is invalid? d=" + d + ", sum=" + sum);
     }
+    private static String getMelodiesFileNameFromURLPath(String midiFileZipFileUrlPath) {
+        if (!(midiFileZipFileUrlPath.endsWith(".zip") || midiFileZipFileUrlPath.endsWith(".ZIP"))) {
+            throw new IllegalStateException("zipFilePath must end with .zip");
+        }
+        midiFileZipFileUrlPath = midiFileZipFileUrlPath.replace('\\','/');
+        String fileName = midiFileZipFileUrlPath.substring(midiFileZipFileUrlPath.lastIndexOf("/") + 1);
+        return fileName + ".txt";
+    }
 }
 

dl4j-examples/src/main/java/org/deeplearning4j/examples/wip/advanced/modelling/melodl4j/MelodyStrings.java

@@ -26,36 +26,102 @@
  *
  * @author Don Smith
  */
+
 public class MelodyStrings {
     public static final String COMMENT_STRING = "//";
     // The following strings are used to build the symbolic representation of a melody
     // The next two strings contain chars used to indicate pitch deltas.
-    public static final String noteGapCharsPositive = "0123456789abc"; // A pitch delta of "0" indicates delta=0.
-    public static final String noteGapCharsNegative = "ABCDEFGHIJKLM"; // A pitch delta of "A" indicates delta=-1.
-    // R is used to indicate the beginning of a rest
+    // We use this ordering of characters so that the pitch gap order is the same as the ASCII order.
+    public static final char lowestPitchGapChar = 'A';
+    public static final char REST_CHAR = ' ';
+
+    // We allow pitch gaps between -12 and +12, inclusive.
+    // If you want to change the allowed gap, you will have to change the characters in PITCH_GAP_CHARS_NEGATIVE and  PITCH_GAP_CHARS_POSITIVE
+
+    public static int MAX_POSITIVE_PITCH_GAP = 12;
+    // The order of characters below is intended to simplify the learning of pitch gaps, because increasing
+    // characters correspond to increasing pitch gaps.  The string must end with 'A'.
+    public static final String PITCH_GAP_CHARS_NEGATIVE = "LKJIHGFEDCBA"; // "L" indicates delta=-1. "K" indicates -2,...
+
+    public static final char FIRST_PITCH_CHAR_NEGATIVE =  PITCH_GAP_CHARS_NEGATIVE.charAt(0);
+
+    // There are thirteen chars in pitchGapCharsPositive because the first one ('M') indicates a zero pitch gap.
+    public static final String PITCH_GAP_CHARS_POSITIVE = "MNOPQRSTUVWXY"; // "M" indicates delta=0. "N" indicates 1, ...
+    public static final char ZERO_PITCH_DELTA_CHAR = 'M';
+    public static final char MAX_POSITIVE_PITCH_DELTA_CHAR = 'Y';
+
+    // durationDeltaParts determines how short durations can get. The shortest duration is 1/durationDeltaParts
+    // as long as the average note length in the piece.
     public static int durationDeltaParts = 8;
-    public static final String durationChars = "defghijkmnopqrstuvwzyz!@#$%^&*-_"; // 32 divisions.  We omit lower-case L to avoid confusion with one.
-    //                                          12345678901234567890123456789012
+    public static final String DURATION_CHARS = "]^_`abcdefghijklmnopqrstuvwxyz{|"; // 32 divisions, in ASCII order
+
+    public static final char FIRST_DURATION_CHAR = ']';
     public static final String allValidCharacters = getValidCharacters();
     // 13+13+1+32 = 59 possible characters.
-    // 'd' indicates the smallest pitch duration allowed (typically a 1/32 note or so).
-    // 'e' is a duration twice that of 'd'
-    // 'f' is a duration three times that of 'd', etc.
-    // If there is a rest between notes, we append 'R' followed by a char for the duration of the rest.
-    public static final char restChar = 'R';
+    // ']' indicates the smallest pitch duration allowed (typically a 1/32 note or so).
+    // '^' is a duration twice that of ']'
+    // '_' is a duration three times that of ']', etc.
+    // If there is a rest between notes, we append ' ' followed by a char for the duration of the rest.
 
     /**
      * @return characters that may occur in a valid melody string
      */
     private static String getValidCharacters() {
         StringBuilder sb = new StringBuilder();
-        sb.append(noteGapCharsPositive);
-        sb.append(noteGapCharsNegative);
-        sb.append(durationChars);
-        sb.append('R');
+        sb.append(PITCH_GAP_CHARS_POSITIVE);
+        sb.append(PITCH_GAP_CHARS_NEGATIVE);
+        sb.append(DURATION_CHARS);
+        sb.append(REST_CHAR);
         return sb.toString();
     }
+    public static boolean isValidMelodyString(String string) {
+        for(int i=0;i<string.length();i++) {
+            if (i%2==0) {
+                if (!isDurationChar(string.charAt(i))) {
+                    return false;
+                }
+            } else {
+                if (!isPitchCharOrRest(string.charAt(i))) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
 
+    public static int getPitchDelta(final char ch) {
+        if (ch >= ZERO_PITCH_DELTA_CHAR && ch <= MAX_POSITIVE_PITCH_DELTA_CHAR) {
+            return ch - ZERO_PITCH_DELTA_CHAR;
+        }
+        if (ch >= 'A' && ch <= FIRST_PITCH_CHAR_NEGATIVE) {
+            return - (1 + (FIRST_PITCH_CHAR_NEGATIVE - ch));
+        }
+        return 0;
+    }
+    public static char getCharForPitchGap(int pitchGap) {
+        while (pitchGap>MAX_POSITIVE_PITCH_GAP) {
+            pitchGap -= MAX_POSITIVE_PITCH_GAP;
+        }
+        while (pitchGap < -MAX_POSITIVE_PITCH_GAP) {
+            pitchGap += MAX_POSITIVE_PITCH_GAP;
+        }
+        return (char) (ZERO_PITCH_DELTA_CHAR + pitchGap);
+    }
+
+    public static int getDurationInTicks(char ch, int resolutionDelta) {
+        int diff = Math.max(0,ch - FIRST_DURATION_CHAR);
+        return diff * resolutionDelta;
+    }
+
+    public static boolean isDurationChar(char ch) {
+        return ch>=']' && ch <= '|';
+    }
+    public static boolean isPitchCharOrRest(char ch) {
+        return ch == REST_CHAR || ch >= 'A' && ch <= 'Z';
+    }
+    public static boolean isPitchChar(char ch) {
+        return ch >= 'A' && ch <= 'Z';
+    }
     public static String convertToMelodyString(List<Note> noteSequence) {
         double averageNoteDuration = computeAverageDuration(noteSequence);
         double durationDelta = averageNoteDuration / durationDeltaParts;
@@ -70,15 +136,15 @@ public static String convertToMelodyString(List<Note> noteSequence) {
                 long restDuration = note.getStartTick() - previousNote.getEndTick();
                 if (restDuration > 0) {
                     char restDurationChar = computeDurationChar(restDuration, durationDelta);
-                    sb.append(restChar);
+                    sb.append(REST_CHAR);
                     sb.append(restDurationChar);
                 }
                 int pitchGap = note.getPitch() - previousNote.getPitch();
-                while (pitchGap >= noteGapCharsPositive.length()) {
-                    pitchGap -= noteGapCharsPositive.length();
+                while (pitchGap > MAX_POSITIVE_PITCH_GAP) {
+                    pitchGap -= MAX_POSITIVE_PITCH_GAP;
                 }
-                while (pitchGap < -noteGapCharsNegative.length()) {
-                    pitchGap += noteGapCharsNegative.length();
+                while (pitchGap < -MAX_POSITIVE_PITCH_GAP) {
+                    pitchGap += MAX_POSITIVE_PITCH_GAP;
                 }
                 sb.append(getCharForPitchGap(pitchGap));
                 long noteDuration = note.getDurationInTicks();
@@ -87,20 +153,25 @@ public static String convertToMelodyString(List<Note> noteSequence) {
             }
             previousNote = note;
         }
-        return sb.toString();
+        String result= sb.toString();
+        if (!isValidMelodyString(result)) {
+            System.err.println("Invalid melody string: " + result);
+        }
+        return result;
     }
 
-    private static char getCharForPitchGap(int pitchGap) {
-        return pitchGap >= 0 ? noteGapCharsPositive.charAt(pitchGap) : noteGapCharsNegative.charAt(-1 - pitchGap);
-    }
 
     private static char computeDurationChar(long duration, double durationDelta) {
-        int times = Math.min((int) Math.round(duration / durationDelta), durationChars.length() - 1);
+        int times = Math.min((int) Math.round(duration / durationDelta), DURATION_CHARS.length() - 1);
         if (times < 0) {
             System.err.println("WARNING: Duration = " + duration);
             times = 0;
         }
-        return durationChars.charAt(times);
+        char ch = DURATION_CHARS.charAt(times);
+        if (!isDurationChar(ch)) {
+            throw new IllegalStateException("Invalid duration char " + ch + " for duration " + duration + ", " + durationDelta);
+        }
+        return ch;
     }
 
     private static double computeAverageDuration(List<Note> noteSequence) {