Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-9.6.x

Signed-off-by: Gavin Halliday <[email protected]>

Author: ghalliday

wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsFileIOClient.java

@@ -232,12 +232,14 @@ public boolean ping() throws Exception
     }
 
     /**
-     * Creates the HPCC file.
+     * @deprecated Due to change in server behavior
+     *             Use boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) instead
+     * Creates an HPCC file.
      *
      * @param fileName
      *            - The target HPCC file name
      * @param targetLandingZone
-     *            - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient
+     *            - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient
      * @param overwritefile
      *            - If the file exists, should it be overwritten?
      * @return true, if successful
@@ -247,17 +249,47 @@ public boolean ping() throws Exception
      *             the array of esp exception wrapper
      */
     public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile) throws Exception, ArrayOfEspExceptionWrapper
+    {
+        return createHPCCFile(fileName, targetLandingZone, overwritefile, null);
+    }
+
+    /**
+     * Creates an HPCC file.
+     *
+     * @param fileName
+     *            - The target HPCC file name
+     * @param targetLandingZone
+     *            - The LZ name, no longer the netaddress of the LZ. should be fetched from landingzones in filesprayclient
+     * @param overwritefile
+     *            - If the file exists, should it be overwritten?
+     * @param lzAddress
+     *            - The Landing zone address
+     * @return true, if successful
+     * @throws java.lang.Exception
+     *             - Caller should handle exception in case of errors
+     * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
+     *             the array of esp exception wrapper
+     */
+    public boolean createHPCCFile(String fileName, String targetLandingZone, boolean overwritefile, String lzAddress) throws Exception, ArrayOfEspExceptionWrapper
     {
         boolean success = false;
         log.debug("Attempting to create HPCC File: " + fileName);
 
+        if (targetLandingZone == null || targetLandingZone.isEmpty())
+            throw new Exception("HPCCWsFileIOClient::createHPCCFile: targetLandingZone required!");
+
+        if (fileName == null || fileName.isEmpty())
+            throw new Exception("HPCCWsFileIOClient::createHPCCFile: fileName required!");
+
         verifyStub(); // Throws exception if stub failed
 
         CreateFileRequest request = new CreateFileRequest();
 
         request.setDestDropZone(targetLandingZone);
         request.setDestRelativePath(fileName);
         request.setOverwrite(overwritefile);
+        if (lzAddress != null && !lzAddress.isEmpty())
+            request.setDestNetAddress(lzAddress);
 
         CreateFileResponse resp = null;
         try
@@ -298,13 +330,15 @@ public boolean createHPCCFile(String fileName, String targetLandingZone, boolean
      *            - At what offset should this be written - Specify 0 if necessary
      * @param uploadchunksize
      *            - Chunksize to upload the data
+     * @param lzAddress
+     *            - The Landing zone address
      * @return true, if successful
      * @throws java.lang.Exception
      *             the exception
      * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
      *             the array of esp exception wrapper
      */
-    public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize)
+    public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress)
             throws Exception, ArrayOfEspExceptionWrapper
     {
         boolean success = true;
@@ -318,6 +352,8 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
         request.setDestDropZone(targetLandingZone);
         request.setDestRelativePath(fileName);
         request.setOffset(offset);
+        if (lzAddress != null && !lzAddress.isEmpty())
+            request.setDestNetAddress(lzAddress);
 
         int dataindex = 0;
         int limit = uploadchunksize <= 0 ? defaultUploadChunkSize : uploadchunksize;
@@ -368,6 +404,35 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
         return success;
     }
 
+    /**
+     * @deprecated Due to change in server behavior
+     *             Use boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize, String lzAddress) instead
+     * Write HPCC file data.
+     *
+     * @param data
+     *            - The data to write
+     * @param fileName
+     *            - The target HPCC file to write to
+     * @param targetLandingZone
+     *            - The "netaddress" of the target landing, can be localhost, should be fetched from landingzones in filesprayclient
+     * @param append
+     *            - Should this data be appended?
+     * @param offset
+     *            - At what offset should this be written - Specify 0 if necessary
+     * @param uploadchunksize
+     *            - Chunksize to upload the data
+     * @return true, if successful
+     * @throws java.lang.Exception
+     *             the exception
+     * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
+     *             the array of esp exception wrapper
+     */
+    public boolean writeHPCCFileData(byte[] data, String fileName, String targetLandingZone, boolean append, long offset, int uploadchunksize)
+            throws Exception, ArrayOfEspExceptionWrapper
+    {
+        return writeHPCCFileData(data, fileName, targetLandingZone, append, offset, uploadchunksize, null);
+    }
+
     /**
      * Read file data.
      *
@@ -379,19 +444,23 @@ public boolean writeHPCCFileData(byte[] data, String fileName, String targetLand
      *            the datasize
      * @param offset
      *            the offset
+     * @param dropzoneAddress
+     *            the dropzone address (not needed in containerized mode)
      * @return the string
      * @throws java.lang.Exception
      *             the exception
      * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
      *             the array of esp exception wrapper
      */
-    public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper
+    public String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) throws Exception, ArrayOfEspExceptionWrapper
     {
         ReadFileDataRequest readFileDataRequest = new ReadFileDataRequest();
         readFileDataRequest.setDestDropZone(dropzone);
         readFileDataRequest.setDestRelativePath(fileName);
         readFileDataRequest.setDataSize(datasize);
         readFileDataRequest.setOffset(offset);
+        if (dropzoneAddress != null && !dropzoneAddress.isEmpty())
+            readFileDataRequest.setDestNetAddress(dropzoneAddress);
 
         ReadFileDataResponse resp = null;
         try
@@ -426,4 +495,28 @@ public String readFileData(String dropzone, String fileName, long datasize, long
 
         return data;
     }
+
+    /**
+     * @deprecated Due to change in server behavior
+     *             Use String readFileData(String dropzone, String fileName, long datasize, long offset, String dropzoneAddress) instead
+     * Read file data.
+     *
+     * @param dropzone
+     *            the dropzone
+     * @param fileName
+     *            the file name
+     * @param datasize
+     *            the datasize
+     * @param offset
+     *            the offset
+     * @return the string
+     * @throws java.lang.Exception
+     *             the exception
+     * @throws org.hpccsystems.ws.client.wrappers.ArrayOfEspExceptionWrapper
+     *             the array of esp exception wrapper
+     */
+    public String readFileData(String dropzone, String fileName, long datasize, long offset) throws Exception, ArrayOfEspExceptionWrapper
+    {
+        return readFileData(dropzone, fileName, datasize, offset, null);
+    }
 }

wsclient/src/test/java/org/hpccsystems/ws/client/WSFileIOClientTest.java

@@ -43,14 +43,24 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 public class WSFileIOClientTest extends BaseRemoteTest
 {
     private final static HPCCWsFileIOClient client = wsclient.getWsFileIOClient();
-
+    private static boolean isContainerized = false;
     private final static String testfilename = System.getProperty("lztestfile", "myfilename.txt");
-    private final static  String targetLZ = System.getProperty("lzname", "localhost");
+    private final static String targetLZ = System.getProperty("lzname", "mydropzone"); //targetLZ accepted the address "localhost" once upon a time.
     private final static String targetLZPath = System.getProperty("lzpath", "/var/lib/HPCCSystems/mydropzone");
-    private final static String HPCC_30117 = System.getProperty("HPCC30117", "fixed");
+    private final static String targetLZAddress = System.getProperty("lzaddress", ".");
 
     static
     {
+        try
+        {
+            if (client.isTargetHPCCContainerized())
+                isContainerized = true;
+        }
+        catch (Exception e)
+        {
+            System.out.println("Could not determine if target service is containerized, default: 'false'");
+        }
+
         if (System.getProperty("lztestfile") == null)
             System.out.println("lztestfile not provided - defaulting to myfilename.txt");
 
@@ -60,20 +70,18 @@ public class WSFileIOClientTest extends BaseRemoteTest
         if (System.getProperty("lzpath") == null)
             System.out.println("lzpath not provided - defaulting to /var/lib/HPCCSystems/mydropzone");
 
-        if (System.getProperty("HPCC30117") == null)
-            System.out.println("HPCC30117 status not provided - defaulting to fixed");
-        else
-            System.out.println("HPCC30117 status: '" + HPCC_30117 + "'");
+        if (System.getProperty("lzaddress") == null)
+            System.out.println("lzaddress not provided - defaulting to '.'");
     }
 
     @Test
     public void copyFile() throws Exception
     {
         String lzfile=System.currentTimeMillis() + "_csvtest.csv";
         String hpccfilename="temp::" + lzfile;
-        client.createHPCCFile(lzfile, targetLZ, true);
+        client.createHPCCFile(lzfile, targetLZ, true, isContainerized ? null : targetLZAddress);
         byte[] data = "Product,SKU,Color\r\nBike,1234,Blue\r\nCar,2345,Red\r\n".getBytes();
-        client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20);
+        client.writeHPCCFileData(data, lzfile, targetLZ, true, 0, 20, isContainerized ? null : targetLZAddress);
         try
         {
             System.out.println("Starting file spray.");
@@ -152,23 +160,49 @@ public void copyFile() throws Exception
     @Test
     public void AcreateHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
     {
-        System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
-        Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true));
+        if (isContainerized)
+        {
+            System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
+            System.out.println("Target HPCC is containerized, not providing targetLZAddress");
+            Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, null));
+        }
+        else
+        {
+            System.out.println("Creating file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' targetLZaddress: '" + targetLZAddress + "' on HPCC: '" + super.connString +"'");
+            System.out.println("Target HPCC is NOT containerized, providing targetLZAddress");
+            Assert.assertTrue(client.createHPCCFile(testfilename, targetLZ, true, targetLZAddress));
+        }
     }
 
     @Test
     public void BwriteHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
     {
+        System.out.println("Writing data to file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
         byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes();
-        Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20));
+        if (isContainerized)
+        {
+            Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, null));
+        }
+        else
+        {
+            Assert.assertTrue(client.writeHPCCFileData(data, testfilename, targetLZ, true, 0, 20, targetLZAddress));
+        }
     }
 
     @Test
     public void CreadHPCCFile() throws Exception, ArrayOfEspExceptionWrapper
     {
         System.out.println("reading data from file: '" + testfilename + "' on LandingZone: '" + targetLZ + "' on HPCC: '" + super.connString +"'");
         byte[] data = "HELLO MY DARLING, HELLO MY DEAR!1234567890ABCDEFGHIJKLMNOPQRSTUVXYZ".getBytes();
-        String response = client.readFileData(targetLZ, testfilename, data.length, 0);
+        String response = null;
+        if (isContainerized)
+        {
+             response = client.readFileData(targetLZ, testfilename, data.length, 0, null);
+        }
+        else
+        {
+             response = client.readFileData(targetLZ, testfilename, data.length, 0, targetLZAddress);
+        }
         Assert.assertNotNull(response);
         Assert.assertArrayEquals(data, response.getBytes());
     }