Added TF2 California housing local training and serving

Author: eitansela

tensorflow_script_mode_california_housing_local_training_and_serving/code/california_housing_tf2.py

@@ -0,0 +1,82 @@
+import argparse
+import numpy as np
+import os
+import tensorflow as tf
+
+def parse_args():
+
+    parser = argparse.ArgumentParser()
+
+    # hyperparameters sent by the client are passed as command-line arguments to the script
+    parser.add_argument('--epochs', type=int, default=1)
+    parser.add_argument('--batch_size', type=int, default=64)
+    parser.add_argument('--learning_rate', type=float, default=0.1)
+
+    # data directories
+    parser.add_argument('--train', type=str, default=os.environ.get('SM_CHANNEL_TRAIN'))
+    parser.add_argument('--test', type=str, default=os.environ.get('SM_CHANNEL_TEST'))
+
+    # model directory
+    parser.add_argument('--sm-model-dir', type=str, default=os.environ.get('SM_MODEL_DIR'))
+
+    return parser.parse_known_args()
+
+
+def get_train_data(train_dir):
+
+    x_train = np.load(os.path.join(train_dir, 'x_train.npy'))
+    y_train = np.load(os.path.join(train_dir, 'y_train.npy'))
+    print('x train', x_train.shape,'y train', y_train.shape)
+
+    return x_train, y_train
+
+
+def get_test_data(test_dir):
+
+    x_test = np.load(os.path.join(test_dir, 'x_test.npy'))
+    y_test = np.load(os.path.join(test_dir, 'y_test.npy'))
+    print('x test', x_test.shape,'y test', y_test.shape)
+
+    return x_test, y_test
+
+
+def get_model():
+
+    inputs = tf.keras.Input(shape=(8,))
+    hidden_1 = tf.keras.layers.Dense(8, activation='tanh')(inputs)
+    hidden_2 = tf.keras.layers.Dense(4, activation='sigmoid')(hidden_1)
+    outputs = tf.keras.layers.Dense(1)(hidden_2)
+    return tf.keras.Model(inputs=inputs, outputs=outputs)
+
+
+if __name__ == "__main__":
+
+    args, _ = parse_args()
+
+    print('Training data location: {}'.format(args.train))
+    print('Test data location: {}'.format(args.test))
+    x_train, y_train = get_train_data(args.train)
+    x_test, y_test = get_test_data(args.test)
+
+    batch_size = args.batch_size
+    epochs = args.epochs
+    learning_rate = args.learning_rate
+    print('batch_size = {}, epochs = {}, learning rate = {}'.format(batch_size, epochs, learning_rate))
+
+
+    model = get_model()
+    optimizer = tf.keras.optimizers.SGD(learning_rate)
+    model.compile(optimizer=optimizer, loss='mse')
+    model.fit(x_train,
+              y_train,
+              batch_size=batch_size,
+              epochs=epochs,
+              validation_data=(x_test, y_test))
+
+    # evaluate on test set
+    scores = model.evaluate(x_test, y_test, batch_size, verbose=2)
+    print("\nTest MSE :", scores)
+
+    # save model
+    model.save(args.sm_model_dir + '/1')
+

tensorflow_script_mode_california_housing_local_training_and_serving/requirements.txt

@@ -0,0 +1,4 @@
+numpy
+pandas
+sagemaker>=2.0.0<3.0.0
+sagemaker[local]

tensorflow_script_mode_california_housing_local_training_and_serving/tensorflow_script_mode_california_housing_local_training_and_serving.py

@@ -0,0 +1,106 @@
+# This is a sample Python program that trains a simple TensorFlow CIFAR-10 model.
+# This implementation will work on your *local computer* or in the *AWS Cloud*.
+# To run training and inference *locally* set: `config = get_config(LOCAL_MODE)`
+# To run training and inference on the *cloud* set: `config = get_config(CLOUD_MODE)` and set a valid IAM role value in get_config()
+#
+# Prerequisites:
+#   1. Install required Python packages:
+#      `pip install -r requirements.txt`
+#   2. Docker Desktop installed and running on your computer:
+#      `docker ps`
+#   3. You should have AWS credentials configured on your local machine
+#      in order to be able to pull the docker image from ECR.
+###############################################################################################
+
+import os
+
+import numpy as np
+import pandas as pd
+from sklearn.datasets import *
+import sklearn.model_selection
+from sklearn.preprocessing import StandardScaler
+from sagemaker.tensorflow import TensorFlow
+
+
+DUMMY_IAM_ROLE = 'arn:aws:iam::111111111111:role/service-role/AmazonSageMaker-ExecutionRole-20200101T000001'
+
+
+def download_training_and_eval_data():
+    if os.path.isfile('./data/train/x_train.npy') and \
+            os.path.isfile('./data/test/x_test.npy') and \
+            os.path.isfile('./data/train/y_train.npy') and \
+            os.path.isfile('./data/test/y_test.npy'):
+        print('Training and evaluation datasets exist. Skipping Download')
+    else:
+        print('Downloading training and evaluation dataset')
+        data_dir = os.path.join(os.getcwd(), 'data')
+        os.makedirs(data_dir, exist_ok=True)
+
+        train_dir = os.path.join(os.getcwd(), 'data/train')
+        os.makedirs(train_dir, exist_ok=True)
+
+        test_dir = os.path.join(os.getcwd(), 'data/test')
+        os.makedirs(test_dir, exist_ok=True)
+
+        data_set = fetch_california_housing()
+
+        X = pd.DataFrame(data_set.data, columns=data_set.feature_names)
+        Y = pd.DataFrame(data_set.target)
+
+        # We partition the dataset into 2/3 training and 1/3 test set.
+        x_train, x_test, y_train, y_test = sklearn.model_selection.train_test_split(X, Y, test_size=0.33)
+
+        scaler = StandardScaler()
+        scaler.fit(x_train)
+        x_train = scaler.transform(x_train)
+        x_test = scaler.transform(x_test)
+
+        np.save(os.path.join(train_dir, 'x_train.npy'), x_train)
+        np.save(os.path.join(test_dir, 'x_test.npy'), x_test)
+        np.save(os.path.join(train_dir, 'y_train.npy'), y_train)
+        np.save(os.path.join(test_dir, 'y_test.npy'), y_test)
+
+        print('Downloading completed')
+
+
+def do_inference_on_local_endpoint(predictor):
+    print(f'\nStarting Inference on endpoint (local).')
+
+    x_test = np.load('./data/test/x_test.npy')
+    y_test = np.load('./data/test/y_test.npy')
+
+    results = predictor.predict(x_test[:10])['predictions']
+    flat_list = [float('%.1f' % (item)) for sublist in results for item in sublist]
+    print('predictions: \t{}'.format(np.array(flat_list)))
+    print('target values: \t{}'.format(y_test[:10].round(decimals=1)))
+
+
+def main():
+    download_training_and_eval_data()
+
+    print('Starting model training.')
+    print(
+        'Note: if launching for the first time in local mode, container image download might take a few minutes to complete.')
+    california_housing_estimator = TensorFlow(entry_point='california_housing_tf2.py',
+                                              source_dir='code',
+                                              role=DUMMY_IAM_ROLE,
+                                              instance_count=1,
+                                              instance_type='local',
+                                              framework_version='2.4.1',
+                                              py_version='py37')
+
+    inputs = {'train': 'file://./data/train', 'test': 'file://./data/test'}
+    california_housing_estimator.fit(inputs)
+    print('Completed model training')
+
+    print('Deploying endpoint in local mode')
+    predictor = california_housing_estimator.deploy(initial_instance_count=1, instance_type='local')
+
+    do_inference_on_local_endpoint(predictor)
+
+    print('About to delete the endpoint to stop paying (if in cloud mode).')
+    predictor.delete_endpoint(predictor.endpoint_name)
+
+
+if __name__ == "__main__":
+    main()