|
| 1 | +import os |
| 2 | +import tempfile |
| 3 | + |
| 4 | +import torch |
| 5 | +from torch.utils.data import DataLoader |
| 6 | +from torchvision.models import resnet18 |
| 7 | +from torchvision.datasets import FashionMNIST |
| 8 | +from torchvision.transforms import ToTensor, Normalize, Compose |
| 9 | +import lightning.pytorch as pl |
| 10 | + |
| 11 | +import ray.train.lightning |
| 12 | +from ray.train.torch import TorchTrainer |
| 13 | + |
| 14 | +# Based on https://docs.ray.io/en/latest/train/getting-started-pytorch-lightning.html |
| 15 | + |
| 16 | +""" |
| 17 | +# For S3 persistent storage replace the following environment variables with your AWS credentials then uncomment the S3 run_config |
| 18 | +# See here for information on how to set up an S3 bucket https://docs.aws.amazon.com/AmazonS3/latest/userguide/creating-bucket.html |
| 19 | +
|
| 20 | +os.environ["AWS_ACCESS_KEY_ID"] = "XXXXXXXX" |
| 21 | +os.environ["AWS_SECRET_ACCESS_KEY"] = "XXXXXXXX" |
| 22 | +os.environ["AWS_DEFAULT_REGION"] = "XXXXXXXX" |
| 23 | +""" |
| 24 | + |
| 25 | +""" |
| 26 | +# For Minio persistent storage uncomment the following code and fill in the name, password and API URL then uncomment the minio run_config. |
| 27 | +# See here for information on how to set up a minio bucket https://ai-on-openshift.io/tools-and-applications/minio/minio/ |
| 28 | +
|
| 29 | +def get_minio_run_config(): |
| 30 | + import s3fs |
| 31 | + import pyarrow.fs |
| 32 | +
|
| 33 | + s3_fs = s3fs.S3FileSystem( |
| 34 | + key = os.getenv('MINIO_ACCESS_KEY', "XXXXX"), |
| 35 | + secret = os.getenv('MINIO_SECRET_ACCESS_KEY', "XXXXX"), |
| 36 | + endpoint_url = os.getenv('MINIO_URL', "XXXXX") |
| 37 | + ) |
| 38 | +
|
| 39 | + custom_fs = pyarrow.fs.PyFileSystem(pyarrow.fs.FSSpecHandler(s3_fs)) |
| 40 | +
|
| 41 | + run_config = ray.train.RunConfig(storage_path='training', storage_filesystem=custom_fs) |
| 42 | + return run_config |
| 43 | +""" |
| 44 | + |
| 45 | + |
| 46 | +# Model, Loss, Optimizer |
| 47 | +class ImageClassifier(pl.LightningModule): |
| 48 | + def __init__(self): |
| 49 | + super(ImageClassifier, self).__init__() |
| 50 | + self.model = resnet18(num_classes=10) |
| 51 | + self.model.conv1 = torch.nn.Conv2d( |
| 52 | + 1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False |
| 53 | + ) |
| 54 | + self.criterion = torch.nn.CrossEntropyLoss() |
| 55 | + |
| 56 | + def forward(self, x): |
| 57 | + return self.model(x) |
| 58 | + |
| 59 | + def training_step(self, batch, batch_idx): |
| 60 | + x, y = batch |
| 61 | + outputs = self.forward(x) |
| 62 | + loss = self.criterion(outputs, y) |
| 63 | + self.log("loss", loss, on_step=True, prog_bar=True) |
| 64 | + return loss |
| 65 | + |
| 66 | + def configure_optimizers(self): |
| 67 | + return torch.optim.Adam(self.model.parameters(), lr=0.001) |
| 68 | + |
| 69 | + |
| 70 | +def train_func(): |
| 71 | + # Data |
| 72 | + transform = Compose([ToTensor(), Normalize((0.5,), (0.5,))]) |
| 73 | + data_dir = os.path.join(tempfile.gettempdir(), "data") |
| 74 | + train_data = FashionMNIST( |
| 75 | + root=data_dir, train=True, download=True, transform=transform |
| 76 | + ) |
| 77 | + train_dataloader = DataLoader(train_data, batch_size=128, shuffle=True) |
| 78 | + |
| 79 | + # Training |
| 80 | + model = ImageClassifier() |
| 81 | + # [1] Configure PyTorch Lightning Trainer. |
| 82 | + trainer = pl.Trainer( |
| 83 | + max_epochs=10, |
| 84 | + devices="auto", |
| 85 | + accelerator="auto", |
| 86 | + strategy=ray.train.lightning.RayDDPStrategy(), |
| 87 | + plugins=[ray.train.lightning.RayLightningEnvironment()], |
| 88 | + callbacks=[ray.train.lightning.RayTrainReportCallback()], |
| 89 | + # [1a] Optionally, disable the default checkpointing behavior |
| 90 | + # in favor of the `RayTrainReportCallback` above. |
| 91 | + enable_checkpointing=False, |
| 92 | + ) |
| 93 | + trainer = ray.train.lightning.prepare_trainer(trainer) |
| 94 | + trainer.fit(model, train_dataloaders=train_dataloader) |
| 95 | + |
| 96 | + |
| 97 | +# [2] Configure scaling and resource requirements. Set the number of workers to the total number of GPUs on your Ray Cluster. |
| 98 | +scaling_config = ray.train.ScalingConfig(num_workers=3, use_gpu=True) |
| 99 | + |
| 100 | +# [3] Launch distributed training job. |
| 101 | +trainer = TorchTrainer( |
| 102 | + train_func, |
| 103 | + scaling_config=scaling_config, |
| 104 | + # run_config = ray.train.RunConfig(storage_path="s3://BUCKET_NAME/SUB_PATH/", name="unique_run_name") # Uncomment and update the S3 URI for S3 persistent storage. |
| 105 | + # run_config=get_minio_run_config(), # Uncomment for minio persistent storage. |
| 106 | +) |
| 107 | +result: ray.train.Result = trainer.fit() |
| 108 | + |
| 109 | +# [4] Load the trained model. |
| 110 | +with result.checkpoint.as_directory() as checkpoint_dir: |
| 111 | + model = ImageClassifier.load_from_checkpoint( |
| 112 | + os.path.join( |
| 113 | + checkpoint_dir, |
| 114 | + ray.train.lightning.RayTrainReportCallback.CHECKPOINT_NAME, |
| 115 | + ), |
| 116 | + ) |
0 commit comments