re-enable copy of python files, but be careful that the copy is only … (pytorch#14982)

Summary:
…done once

This allow no-op build to work correctly even when BUILD_CAFFE2_OPS is on.
Pull Request resolved: pytorch#14982

Differential Revision: D13413960

Pulled By: zdevito

fbshipit-source-id: 6e5412a8c375af8a47c76f548cdd31cff15f3853

Author: zdevito

caffe2/CMakeLists.txt

@@ -471,7 +471,7 @@ endif()
 # rebuild step. The long-term fix should be to clean up these rules so they
 # only rerun when needed.
 
-if (BUILD_PYTHON AND BUILD_CAFFE2_OPS)
+if (BUILD_PYTHON)
   # Python site-packages
   # Get canonical directory for python site packages (relative to install
   # location).  It varies from system to system.
@@ -628,37 +628,22 @@ if (BUILD_PYTHON AND BUILD_CAFFE2_OPS)
   endif()
 
   # Finally, Copy all python files to build directory
-  # Generate and create all needed __init__.py files, if they aren't already
-  # present in the current source tree.
-  message(STATUS "Automatically generating missing __init__.py files.")
-  caffe_autogen_init_py_files()
-
   # Create a custom target that copies all python files.
   file(GLOB_RECURSE PYTHON_SRCS RELATIVE ${PROJECT_SOURCE_DIR}
        "${PROJECT_SOURCE_DIR}/caffe2/*.py")
-  add_custom_target(python_copy_files ALL)
-  if(MSVC OR CMAKE_GENERATOR MATCHES "Ninja")
-    # ninja fails when the command line is too long so we split
-    # the target into several. This would be beneficial for VS also
-    # since it build targets in parallel but not custom commands
-    foreach(python_src ${PYTHON_SRCS})
-      get_filename_component(dir ${python_src} DIRECTORY)
-      string(SHA1 name_hash "${python_src}")
-      # get_filename_component(name_we ${python_src} NAME_WE)
-      add_custom_target(python_copy_files_${name_hash}
-          COMMAND ${CMAKE_COMMAND} -E copy
-          ${PROJECT_SOURCE_DIR}/${python_src} ${CMAKE_BINARY_DIR}/${dir})
-      add_dependencies(python_copy_files python_copy_files_${name_hash})
-    endforeach()
-  else()
-    foreach(python_src ${PYTHON_SRCS})
-      get_filename_component(dir ${python_src} DIRECTORY)
-      add_custom_command(
-          TARGET python_copy_files PRE_BUILD
-          COMMAND ${CMAKE_COMMAND} -E copy
-          ${PROJECT_SOURCE_DIR}/${python_src} ${CMAKE_BINARY_DIR}/${dir})
-    endforeach()
-  endif()
+
+  set(build_files)
+  foreach(python_src ${PYTHON_SRCS})
+    add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/${python_src}
+                       DEPENDS ${PROJECT_SOURCE_DIR}/${python_src}
+                       COMMAND ${CMAKE_COMMAND} -E copy
+                       ${PROJECT_SOURCE_DIR}/${python_src}
+                       ${CMAKE_BINARY_DIR}/${python_src})
+    list(APPEND build_files ${CMAKE_BINARY_DIR}/${python_src})
+  endforeach()
+
+  add_custom_target(python_copy_files ALL DEPENDS ${build_files})
+
 
   # Install commands
   # Pick up static python files

caffe2/contrib/aten/docs/__init__.py

@@ -128,42 +128,6 @@ function(caffe2_parse_version_str LIBNAME VERSIONSTR)
   set(${LIBNAME}_VERSION "${${LIBNAME}_VERSION_MAJOR}.${${LIBNAME}_VERSION_MINOR}.${${LIBNAME}_VERSION_PATCH}" PARENT_SCOPE)
 endfunction()
 
-##############################################################################
-# Helper function to automatically generate __init__.py files where python
-# sources reside but there are no __init__.py present.
-function(caffe_autogen_init_py_files)
-  if (CAFFE2_AUTOGEN_INIT_PY_ALREADY_RUN)
-    message(STATUS
-        "A previous caffe2 cmake run already created the __init__.py files.")
-    return()
-  endif()
-  file(GLOB_RECURSE all_python_files RELATIVE ${PROJECT_SOURCE_DIR}
-       "${PROJECT_SOURCE_DIR}/caffe2/*.py")
-  set(python_paths_need_init_py)
-  foreach(python_file ${all_python_files})
-    get_filename_component(python_path ${python_file} PATH)
-    string(REPLACE "/" ";" path_parts ${python_path})
-    set(rebuilt_path ${CMAKE_BINARY_DIR})
-    foreach(path_part ${path_parts})
-      set(rebuilt_path "${rebuilt_path}/${path_part}")
-      list(APPEND python_paths_need_init_py ${rebuilt_path})
-    endforeach()
-  endforeach()
-  list(REMOVE_DUPLICATES python_paths_need_init_py)
-  # Since the _pb2.py files are yet to be created, we will need to manually
-  # add them to the list.
-  list(APPEND python_paths_need_init_py ${CMAKE_BINARY_DIR}/caffe2/proto)
-
-  foreach(tmp ${python_paths_need_init_py})
-    if(NOT EXISTS ${tmp}/__init__.py)
-      # message(STATUS "Generate " ${tmp}/__init__.py)
-      file(WRITE ${tmp}/__init__.py "")
-    endif()
-  endforeach()
-  set(CAFFE2_AUTOGEN_INIT_PY_ALREADY_RUN TRUE CACHE INTERNAL
-      "Helper variable to record if autogen_init_py is already run or not.")
-endfunction()
-
 ###
 # Removes common indentation from a block of text to produce code suitable for
 # setting to `python -c`, or using with pycmd. This allows multiline code to be

caffe2/contrib/cuda-convnet2/make-data/__init__.py

@@ -425,18 +425,6 @@ def build_libs(libs):
         sys.exit(1)
 
 
-# Copy Caffe2's Python proto files (generated during the build with the
-# protobuf python compiler) from the build folder to the root folder
-# cp root/build/caffe2/proto/proto.py root/caffe2/proto/proto.py
-def copy_protos():
-    report('setup.py::copy_protos()')
-    for src in glob.glob(
-            os.path.join(caffe2_build_dir, 'caffe2', 'proto', '*.py')):
-        dst = os.path.join(
-            cwd, os.path.relpath(src, caffe2_build_dir))
-        shutil.copyfile(src, dst)
-
-
 # Build all dependent libraries
 class build_deps(PytorchCommand):
     def run(self):
@@ -528,7 +516,6 @@ def run(self):
         self.run_command('create_version_file')
         setuptools.command.develop.develop.run(self)
         self.create_compile_commands()
-        copy_protos()
 
     def create_compile_commands(self):
         def load(filename):

caffe2/contrib/script/examples/__init__.py

@@ -287,7 +287,11 @@ function build_caffe2() {
       fi
 
       for proto_file in $(pwd)/caffe2/proto/*.py; do
-          cp $proto_file "$(pwd)/../caffe2/proto/"
+          # __init__.py is not auto-generated, copying it breaks
+          # mod-times for rebuild logic
+          if [[ "$proto_file" != "$(pwd)/caffe2/proto/__init__.py" ]]; then
+            cp $proto_file "$(pwd)/../caffe2/proto/"
+          fi
       done
   fi