Remove knet. (PR samtools#1200)

There's little benefit to keeping it (basically FTP and unsecured
http), while curl is already required for htsget, refget (CRAM ref
cache), AWS S3 and GCS protocols.  Curl is available on all platforms
we support.

The public API in htslib/knetfile.h is retained for compatibility.  The knet_*
functions have been replaced by stubs that use hFILE internally.
htslib/knetfile.h has HTS_DEPRECATED markers for these functions, so
anyone compiling against the new code will get a warning that we
intend to delete them in the future.

The knet_tell macro does direct access of knetFile::offset so we
cannot simply wrap up htell as we'd like.  Similarly knet_fileno
directly queries knetFile::fd.  However the only use I've found of
this was in old copies of bgzf and bcftools main which then did fstat
on it so it's only filled out for local files.  (That's good because
getting it out of curl isn't supported by our code.)

Author: jkbonfield

INSTALL

@@ -41,9 +41,10 @@ a development ('-dev' or '-devel') package separate from the main library.
 Disabling libbzip2 and liblzma will make some CRAM files unreadable, so
 is not recommended.
 
-Using libcurl provides HTSlib with better network protocol support, for
-example it enables the use of https:// URLs.  It is also required if
-direct access to Amazon S3 or Google Cloud Storage is enabled.
+Using libcurl provides HTSlib with network protocol support, for
+example it enables the use of ftp://, http://, and https:// URLs.
+It is also required if direct access to Amazon S3 or Google Cloud
+Storage is enabled.
 
 Amazon S3 support requires an HMAC function to calculate a message
 authentication code.  On MacOS, the CCHmac function from the standard
@@ -131,8 +132,7 @@ various features and specify further optional external requirements:
 
 --enable-libcurl
     Use libcurl (<http://curl.haxx.se/>) to implement network access to
-    remote files via FTP, HTTP, HTTPS, etc.  By default, HTSlib uses its
-    own simple networking code to provide access via FTP and HTTP only.
+    remote files via FTP, HTTP, HTTPS, etc.
 
 --enable-gcs
     Implement network access to Google Cloud Storage.  By default or with

Makefile

@@ -144,15 +144,13 @@ show-version:
 
 LIBHTS_OBJS = \
 	kfunc.o \
-	knetfile.o \
 	kstring.o \
 	bcf_sr_sort.o \
 	bgzf.o \
 	errmod.o \
 	faidx.o \
 	header.o \
 	hfile.o \
-	hfile_net.o \
 	hts.o \
 	hts_expr.o \
 	hts_os.o\
@@ -334,12 +332,10 @@ hts-object-files: $(LIBHTS_OBJS)
 bgzf.o bgzf.pico: bgzf.c config.h $(htslib_hts_h) $(htslib_bgzf_h) $(htslib_hfile_h) $(htslib_thread_pool_h) $(htslib_hts_endian_h) cram/pooled_alloc.h $(hts_internal_h) $(htslib_khash_h)
 errmod.o errmod.pico: errmod.c config.h $(htslib_hts_h) $(htslib_ksort_h) $(htslib_hts_os_h)
 kstring.o kstring.pico: kstring.c config.h $(htslib_kstring_h)
-knetfile.o knetfile.pico: knetfile.c config.h $(htslib_hts_log_h) $(htslib_knetfile_h)
 header.o header.pico: header.c config.h $(textutils_internal_h) $(header_h)
 hfile.o hfile.pico: hfile.c config.h $(htslib_hfile_h) $(hfile_internal_h) $(htslib_kstring_h) $(hts_internal_h) $(htslib_khash_h)
 hfile_gcs.o hfile_gcs.pico: hfile_gcs.c config.h $(htslib_hts_h) $(htslib_kstring_h) $(hfile_internal_h)
 hfile_libcurl.o hfile_libcurl.pico: hfile_libcurl.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h) $(htslib_khash_h)
-hfile_net.o hfile_net.pico: hfile_net.c config.h $(hfile_internal_h) $(htslib_knetfile_h)
 hfile_s3_write.o hfile_s3_write.pico: hfile_s3_write.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h) $(htslib_khash_h)
 hfile_s3.o hfile_s3.pico: hfile_s3.c config.h $(hfile_internal_h) $(htslib_hts_h) $(htslib_kstring_h)
 hts.o hts.pico: hts.c config.h $(htslib_hts_expr_h) $(htslib_hts_h) $(htslib_bgzf_h) $(cram_h) $(htslib_hfile_h) $(htslib_hts_endian_h) version.h $(hts_internal_h) $(hfile_internal_h) $(sam_internal_h) $(htslib_hts_os_h) $(htslib_khash_h) $(htslib_kseq_h) $(htslib_ksort_h) $(htslib_tbx_h)
@@ -544,8 +540,7 @@ test-shlib-exports: header-exports.txt shlib-exports-$(SHLIB_FLAVOUR).txt
 	( echo "Error: Found unexported symbols (listed above)" ; false )
 
 # Extract symbols that should be exported from public headers using ctags
-# Filter out macros in htslib/hts_defs.h, and knet_win32_ functions that
-# aren't needed on non-Windows platforms.
+# Filter out macros in htslib/hts_defs.h.
 header-exports.txt: test/header_syms.pl htslib/*.h
 	test/header_syms.pl htslib/*.h | sort -u -o $@
 

NEWS

@@ -1,7 +1,13 @@
 Noteworthy changes in release a.b
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+* The knet networking code has been removed.  It only supported the http
+  and ftp protocols, and a better and safer alternative using libcurl
+  has been available since release 1.3.  If you need access to ftp:// and
+  http:// URLs, HTSlib should be built with libcurl support. (#1200)
 
+* The old htslib/knetfile.h interfaces have been marked as deprecated.  Any
+  code still using them should be updated to use hFILE instead. (#1200)
 
 Noteworthy changes in release 1.11 (22nd September 2020)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

hfile.c

@@ -1061,7 +1061,6 @@ static int load_hfile_plugins()
     hfile_add_scheme_handler("data", &data);
     hfile_add_scheme_handler("file", &file);
     hfile_add_scheme_handler("preload", &preload);
-    init_add_plugin(NULL, hfile_plugin_init_net, "knetfile");
     init_add_plugin(NULL, hfile_plugin_init_mem, "mem");
     init_add_plugin(NULL, hfile_plugin_init_crypt4gh_needed, "crypt4gh-needed");
 
@@ -1213,3 +1212,82 @@ char *haddextension(struct kstring_t *buffer, const char *filename,
         kputs(trailing, buffer) >= 0) return buffer->s;
     else return NULL;
 }
+
+
+/*
+ * ----------------------------------------------------------------------
+ * Minimal stub functions for knet, added after the removal of
+ * hfile_net.c and knetfile.c.
+ *
+ * They exist purely for ABI compatibility, but are simply wrappers to
+ * hFILE.  API should be compatible except knet_fileno (unused?).
+ *
+ * CULL THESE and knetfile.h at the next .so version bump.
+ */
+typedef struct knetFile_s {
+    // As per htslib/knetfile.h.  Duplicated here as we don't wish to
+    // have any dependence on the deprecated knetfile.h interface, plus
+    // it's hopefully only temporary.
+    int type, fd;
+    int64_t offset;
+    char *host, *port;
+    int ctrl_fd, pasv_ip[4], pasv_port, max_response, no_reconnect, is_ready;
+    char *response, *retr, *size_cmd;
+    int64_t seek_offset;
+    int64_t file_size;
+    char *path, *http_host;
+
+    // Our local addition
+    hFILE *hf;
+} knetFile;
+
+HTSLIB_EXPORT
+knetFile *knet_open(const char *fn, const char *mode) {
+    knetFile *fp = calloc(1, sizeof(*fp));
+    if (!fp) return NULL;
+    if (!(fp->hf = hopen(fn, mode))) {
+        free(fp);
+        fp = NULL;
+    }
+
+    // FD backend is the only one implementing knet_fileno
+    fp->fd = fp->hf->backend == &fd_backend
+        ? ((hFILE_fd *)fp->hf)->fd
+        : -1;
+
+    return fp;
+}
+
+HTSLIB_EXPORT
+knetFile *knet_dopen(int fd, const char *mode) {
+    knetFile *fp = calloc(1, sizeof(*fp));
+    if (!fp) return NULL;
+    if (!(fp->hf = hdopen(fd, mode))) {
+        free(fp);
+        fp = NULL;
+    }
+    fp->fd = fd;
+    return fp;
+}
+
+HTSLIB_EXPORT
+ssize_t knet_read(knetFile *fp, void *buf, size_t len) {
+    ssize_t r = hread(fp->hf, buf, len);
+    fp->offset += r>0?r:0;
+    return r;
+}
+
+HTSLIB_EXPORT
+off_t knet_seek(knetFile *fp, off_t off, int whence) {
+    off_t r = hseek(fp->hf, off, whence);
+    if (r >= 0)
+        fp->offset = r;
+    return r;
+}
+
+HTSLIB_EXPORT
+int knet_close(knetFile *fp) {
+    int r = hclose(fp->hf);
+    free(fp);
+    return r;
+}

hfile_internal.h

@@ -179,9 +179,6 @@ extern int hfile_plugin_init_s3(struct hFILE_plugin *self);
 extern int hfile_plugin_init_s3_write(struct hFILE_plugin *self);
 #endif
 
-/* This one is never built as a separate plugin.  */
-extern int hfile_plugin_init_net(struct hFILE_plugin *self);
-
 // Callback to allow headers to be set in http connections.  Currently used
 // to allow s3 to renew tokens when seeking.  Kept internal for now,
 // although we may consider exposing it in the API later.

hfile_libcurl.c

@@ -1221,7 +1221,7 @@ libcurl_open(const char *url, const char *modes, http_headers *headers)
     err = curl_easy_setopt(fp->easy, CURLOPT_PRIVATE, fp);
 
     // Avoid many repeated CWD calls with FTP, instead requesting the filename
-    // by full path (as done in knet, but not strictly compliant with RFC1738).
+    // by full path (but not strictly compliant with RFC1738).
     err |= curl_easy_setopt(fp->easy, CURLOPT_FTP_FILEMETHOD, CURLFTPMETHOD_NOCWD);
 
     if (mode == 'r') {

hfile_net.c

@@ -62,7 +62,6 @@ HTSLIB_PUBLIC_HEADERS = \
 	$(HTSDIR)/htslib/khash.h \
 	$(HTSDIR)/htslib/khash_str2int.h \
 	$(HTSDIR)/htslib/klist.h \
-	$(HTSDIR)/htslib/knetfile.h \
 	$(HTSDIR)/htslib/kseq.h \
 	$(HTSDIR)/htslib/ksort.h \
 	$(HTSDIR)/htslib/kstring.h \
@@ -89,15 +88,13 @@ HTSLIB_ALL = \
 	$(HTSDIR)/hfile.c \
 	$(HTSDIR)/hfile_gcs.c \
 	$(HTSDIR)/hfile_libcurl.c \
-	$(HTSDIR)/hfile_net.c \
 	$(HTSDIR)/hfile_s3.c \
 	$(HTSDIR)/hfile_s3_write.c \
 	$(HTSDIR)/hts.c \
 	$(HTSDIR)/hts_expr.c \
 	$(HTSDIR)/hts_internal.h \
 	$(HTSDIR)/hts_os.c \
 	$(HTSDIR)/kfunc.c \
-	$(HTSDIR)/knetfile.c \
 	$(HTSDIR)/kstring.c \
 	$(HTSDIR)/md5.c \
 	$(HTSDIR)/multipart.c \