Skip to content

Commit cf1a378

Browse files
agibsoncccagibsonccc
committed
Add oreilly book examples
1 parent 2602618 commit cf1a378

File tree

694 files changed

+204250
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

694 files changed

+204250
-0
lines changed

‎oreilly-book-dl4j-examples/.editorconfig

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
root = true
2+
3+
[*]
4+
charset = utf-8
5+
end_of_line = lf
6+
insert_final_newline = true
7+
trim_trailing_whitespace = true
8+
9+
[*.java]
10+
indent_style = space
11+
indent_size = 4

‎oreilly-book-dl4j-examples/.gitignore

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
target/
2+
.idea/*
3+
*.iml
4+
logs/
5+
*.ser
6+
*.log
7+
*.ipynb_checkpoints
8+
*.pyc
9+
graph-tmp/
10+
11+
.DS_Store
12+
pom.xml.releaseBackup
13+
14+
dependency-reduced-pom.xml
15+
*.ser
16+
application.home_IS_UNDEFINEiD
17+
README.md~
18+
*.bin
19+
*.releaseBackup
20+
*.bin
21+
*.out
22+
*~
23+
.pydevproject
24+
release.properties
25+
.idea/
26+
*.iml
27+
*.prefs
28+
*.settings/*
29+
model-saver-*
30+
mnist-pretrain-dbn.bin-*
31+
deeplearning4j-scaleout/deeplearning4j-aws/src/main/java/org/deeplearning4j/aws/ec2/provision/Ec2CommandRunner.java.settings/
32+
*.log
33+
.project
34+
.classpath
35+
metastore_db
36+
*.ipynb*
37+
*.html
38+
Word2vec-index/
39+
*.conf
40+
*.json
41+
*end.model
42+
dl4j-memory-crash-dump*.txt
43+
44+
CustomLayerModel.zip
45+
MyComputationGraph.zip
46+
MyMultiLayerNetwork.zip
47+
arbiterExample/
48+
dl4j-examples/src/main/resources/PredictGender/DataPredictGender.net
49+
pathToSaveModel.txt
50+
pathToWriteto.txt

‎oreilly-book-dl4j-examples/LICENSE.txt

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Copyright 2016 Skymind Inc.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.

‎oreilly-book-dl4j-examples/README.md

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
Deeplearning4J Examples
2+
=========================
3+
(NOTE TO READERS: THESE EXAMPLES ARE PRESENT FOR HISTORICAL REASONS ONLY. PLEASE USE THE OTHER EXAMPLES IN THIS REPOSITORY FOR THE MOST UP TO DATE CODE.)
4+
Repository of Deeplearning4J neural net examples:
5+
6+
- MLP Neural Nets
7+
- Convolutional Neural Nets
8+
- Recurrent Neural Nets
9+
- TSNE
10+
- Word2Vec & GloVe
11+
- Anomaly Detection
12+
13+
---
14+
15+
## Build and Run
16+
17+
Use [Maven](https://maven.apache.org/) to build the examples.
18+
19+
```
20+
mvn clean package
21+
```
22+
23+
Run the `runexamples.sh` script to run the examples (requires [bash](https://www.gnu.org/software/bash/)). It will list the examples and prompt you for the one to run. Pass the `--all` argument to run all of them. (Other options are shown with `-h`).
24+
25+
```
26+
./runexamples.sh [-h | --help]
27+
```
28+
29+
30+
## Documentation
31+
For more information, check out [deeplearning4j.org](http://deeplearning4j.org/) and its [JavaDoc](http://deeplearning4j.org/doc/).
32+
33+
If you notice issues, please log them, and if you want to contribute, submit a pull request. Input is welcome here.
34+
35+

‎oreilly-book-dl4j-examples/datavec-examples/pom.xml

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<modelVersion>4.0.0</modelVersion>
6+
<artifactId>datavec-examples</artifactId>
7+
8+
<parent>
9+
<groupId>org.deeplearning4j</groupId>
10+
<artifactId>oreilly-book-deeplearning4j-examples-parent</artifactId>
11+
<version>1.0.0-beta2-SNAPSHOT</version>
12+
</parent>
13+
14+
<dependencies>
15+
<dependency>
16+
<groupId>org.datavec</groupId>
17+
<artifactId>datavec-api</artifactId>
18+
<version>${datavec.version}</version>
19+
</dependency>
20+
21+
<dependency>
22+
<groupId>org.datavec</groupId>
23+
<artifactId>datavec-spark_${scala.binary.version}</artifactId>
24+
<version>${datavec.version}_spark_2</version>
25+
</dependency>
26+
<dependency>
27+
<!-- Force Jackson versions to match Spark. This usually only matters for Spark local, not Spark submit -->
28+
<groupId>com.fasterxml.jackson.core</groupId>
29+
<artifactId>jackson-core</artifactId>
30+
<version>${jackson.version}</version>
31+
</dependency>
32+
<dependency>
33+
<groupId>com.fasterxml.jackson.core</groupId>
34+
<artifactId>jackson-databind</artifactId>
35+
<version>${jackson.version}</version>
36+
</dependency>
37+
<dependency>
38+
<groupId>com.fasterxml.jackson.core</groupId>
39+
<artifactId>jackson-annotations</artifactId>
40+
<version>${jackson.version}</version>
41+
</dependency>
42+
<dependency>
43+
<groupId>com.fasterxml.jackson.module</groupId>
44+
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
45+
<version>${jackson.version}</version>
46+
</dependency>
47+
<dependency>
48+
<groupId>com.fasterxml.jackson.datatype</groupId>
49+
<artifactId>jackson-datatype-jdk8</artifactId>
50+
<version>${jackson.version}</version>
51+
</dependency>
52+
<dependency>
53+
<groupId>com.fasterxml.jackson.datatype</groupId>
54+
<artifactId>jackson-datatype-jsr310</artifactId>
55+
<version>${jackson.version}</version>
56+
</dependency>
57+
</dependencies>
58+
59+
<build>
60+
<plugins>
61+
<plugin>
62+
<groupId>org.codehaus.mojo</groupId>
63+
<artifactId>exec-maven-plugin</artifactId>
64+
<version>${exec-maven-plugin.version}</version>
65+
<executions>
66+
<execution>
67+
<goals>
68+
<goal>exec</goal>
69+
</goals>
70+
</execution>
71+
</executions>
72+
<configuration>
73+
<executable>java</executable>
74+
</configuration>
75+
</plugin>
76+
<plugin>
77+
<groupId>org.apache.maven.plugins</groupId>
78+
<artifactId>maven-shade-plugin</artifactId>
79+
<version>${maven-shade-plugin.version}</version>
80+
<configuration>
81+
<shadedArtifactAttached>true</shadedArtifactAttached>
82+
<shadedClassifierName>${shadedClassifier}</shadedClassifierName>
83+
<createDependencyReducedPom>true</createDependencyReducedPom>
84+
<filters>
85+
<filter>
86+
<artifact>*:*</artifact>
87+
<excludes>
88+
<exclude>org/datanucleus/**</exclude>
89+
<exclude>META-INF/*.SF</exclude>
90+
<exclude>META-INF/*.DSA</exclude>
91+
<exclude>META-INF/*.RSA</exclude>
92+
</excludes>
93+
</filter>
94+
</filters>
95+
</configuration>
96+
<executions>
97+
<execution>
98+
<phase>package</phase>
99+
<goals>
100+
<goal>shade</goal>
101+
</goals>
102+
<configuration>
103+
<transformers>
104+
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
105+
<resource>reference.conf</resource>
106+
</transformer>
107+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
108+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
109+
</transformer>
110+
</transformers>
111+
</configuration>
112+
</execution>
113+
</executions>
114+
</plugin>
115+
116+
<plugin>
117+
<groupId>org.apache.maven.plugins</groupId>
118+
<artifactId>maven-compiler-plugin</artifactId>
119+
<version>3.5.1</version>
120+
<configuration>
121+
<source>${java.version}</source>
122+
<target>${java.version}</target>
123+
</configuration>
124+
</plugin>
125+
</plugins>
126+
</build>
127+
128+
</project>

‎oreilly-book-dl4j-examples/datavec-examples/src/main/java/org/datavec/transform/analysis/IrisAnalysis.java

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package org.datavec.transform.analysis;
2+
3+
import org.apache.spark.SparkConf;
4+
import org.apache.spark.api.java.JavaRDD;
5+
import org.apache.spark.api.java.JavaSparkContext;
6+
import org.datavec.api.records.reader.RecordReader;
7+
import org.datavec.api.records.reader.impl.csv.CSVRecordReader;
8+
import org.datavec.api.transform.DataAction;
9+
import org.datavec.api.transform.analysis.DataAnalysis;
10+
import org.datavec.api.transform.analysis.columns.DoubleAnalysis;
11+
import org.datavec.api.transform.schema.Schema;
12+
import org.datavec.api.transform.ui.HtmlAnalysis;
13+
import org.datavec.api.util.ClassPathResource;
14+
import org.datavec.api.writable.Writable;
15+
import org.datavec.spark.transform.AnalyzeSpark;
16+
import org.datavec.spark.transform.misc.StringToWritablesFunction;
17+
import org.datavec.spark.transform.utils.SparkUtils;
18+
19+
import java.io.File;
20+
import java.util.List;
21+
22+
/**
23+
* Conduct and export some basic analysis on the Iris dataset, as a stand-alone .html file.
24+
*
25+
* This functionality is still fairly basic, but can still be useful for analysis and debugging.
26+
*
27+
* @author Alex Black
28+
*/
29+
public class IrisAnalysis {
30+
31+
public static void main(String[] args) throws Exception {
32+
33+
Schema schema = new Schema.Builder()
34+
.addColumnsDouble("Sepal length", "Sepal width", "Petal length", "Petal width")
35+
.addColumnInteger("Species")
36+
.build();
37+
38+
SparkConf conf = new SparkConf();
39+
conf.setMaster("local[*]");
40+
conf.setAppName("DataVec Example");
41+
42+
JavaSparkContext sc = new JavaSparkContext(conf);
43+
44+
String directory = new ClassPathResource("IrisData/iris.txt").getFile().getParent(); //Normally just define your directory like "file:/..." or "hdfs:/..."
45+
JavaRDD<String> stringData = sc.textFile(directory);
46+
47+
//We first need to parse this comma-delimited (CSV) format; we can do this using CSVRecordReader:
48+
RecordReader rr = new CSVRecordReader();
49+
JavaRDD<List<Writable>> parsedInputData = stringData.map(new StringToWritablesFunction(rr));
50+
51+
int maxHistogramBuckets = 10;
52+
DataAnalysis dataAnalysis = AnalyzeSpark.analyze(schema, parsedInputData, maxHistogramBuckets);
53+
54+
System.out.println(dataAnalysis);
55+
56+
//We can get statistics on a per-column basis:
57+
DoubleAnalysis da = (DoubleAnalysis)dataAnalysis.getColumnAnalysis("Sepal length");
58+
double minValue = da.getMin();
59+
double maxValue = da.getMax();
60+
double mean = da.getMean();
61+
62+
HtmlAnalysis.createHtmlAnalysisFile(dataAnalysis, new File("DataVecIrisAnalysis.html"));
63+
64+
//To write to HDFS instead:
65+
//String htmlAnalysisFileContents = HtmlAnalysis.createHtmlAnalysisString(dataAnalysis);
66+
//SparkUtils.writeStringToFile("hdfs://your/hdfs/path/here",htmlAnalysisFileContents,sc);
67+
}
68+
69+
70+
}

0 commit comments

Comments
 (0)