Request for Comments-32: Shark ML integration

From OTBWiki
Jump to: navigation, search

[Request for Comments - 32]: Shark ML integration

Status

  • Author: Jordi Inglada
  • Submitted on 09.06.2016
  • Open for comments

Content

What changes will be made and why they would make a better Orfeo ToolBox?

What's wrong with OpenCV's ML?

Not much: thanks to OpenCV OTB has a very rich set of ML algorithms, but there are some annoying things:

  • reliability of some algorithms: the linear SVM case
  • no hooks available for accessing useful information: confidence in RF
  • learning is not parallel
About Shark
  • Shark is a fast, modular, feature-rich open-source C++ machine learning library.
  • It provides methods for linear and nonlinear optimisation, kernel-based learning algorithms, neural networks, and various other machine learning techniques.
  • It serves as a powerful toolbox for real world applications as well as research.
  • Shark depends on Boost and CMake. It is compatible with Windows, Solaris, MacOS X, and Linux.
  • Shark is licensed under the permissive GNU Lesser General Public License.
  • http://image.diku.dk/shark/index.html

Shark uses OpenMP for parallel learning and prediction and makes extensive use of data batches for data locality and therefore efficiency.

Expected improvements in OTB
  • Richer set of algorithms available and possibility of combining them into more complex models
  • More efficient computation than current OTB Machine Learning Models
Changes to OTB

Update: All the code cited below has been moved to a separate module named SharkLearning and all changes to existing OTB classes have been rolled back. This allows to easily experiment the solutions for the problems listed below.

Adding a new dependency to OTB

A new ThirdParty module needs to be added:

diff --git a/Modules/ThirdParty/Shark/CMakeLists.txt b/Modules/ThirdParty/Shark/CMakeLists.txt
deleted file mode 100644
index 19f8563..0000000
--- a/Modules/ThirdParty/Shark/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-project(OTBShark)
-
-set(OTBShark_SYSTEM_INCLUDE_DIRS ${SHARK_INCLUDE_DIRS})
-set(OTBShark_LIBRARIES "${SHARK_LIBS}")
-
-otb_module_impl()
diff --git a/Modules/ThirdParty/Shark/otb-module-init.cmake b/Modules/ThirdParty/Shark/otb-module-init.cmake
deleted file mode 100644
index fdfb6b8..0000000
--- a/Modules/ThirdParty/Shark/otb-module-init.cmake
+++ /dev/null
@@ -1,2 +0,0 @@
-find_package ( Shark REQUIRED )
-mark_as_advanced( Shark_DIR )
diff --git a/Modules/ThirdParty/Shark/otb-module.cmake b/Modules/ThirdParty/Shark/otb-module.cmake
deleted file mode 100644
index 5d33d0f..0000000
--- a/Modules/ThirdParty/Shark/otb-module.cmake
+++ /dev/null
@@ -1,12 +0,0 @@
-set(DOCUMENTATION "This module imports SHARK to the build system")
-
-otb_module(OTBShark
-  DEPENDS
-    
-  TEST_DEPENDS
-    
-  DESCRIPTION
-    "${DOCUMENTATION}"
-  )
-
-otb_module_activation_option("Enable SHARK dependent modules" OFF)

Since Shark is not available as a package, it has to be added to the Superbuild

diff --git a/SuperBuild/CMake/External_otb.cmake b/SuperBuild/CMake/External_otb.cmake
index 02cd44d..135ca6b 100644
--- a/SuperBuild/CMake/External_otb.cmake
+++ b/SuperBuild/CMake/External_otb.cmake
@@ -39,10 +39,6 @@ if(OTB_USE_OPENCV)
 ADDTO_DEPENDENCIES_IF_NOT_SYSTEM(OTB OPENCV)
 endif()
 
-if(OTB_USE_SHARK)
-  ADDTO_DEPENDENCIES_IF_NOT_SYSTEM(OTB SHARK)
-endif()
-
 if(OTB_USE_LIBSVM)
 ADDTO_DEPENDENCIES_IF_NOT_SYSTEM(OTB LIBSVM)
 endif()
@@ -105,7 +101,6 @@ ADD_SUPERBUILD_CMAKE_VAR(OTB LIBKML_XSD_LIBRARY)
 ADD_SUPERBUILD_CMAKE_VAR(OTB LIBKML_MINIZIP_LIBRARY)
 
 ADD_SUPERBUILD_CMAKE_VAR(OTB OpenCV_DIR)
-ADD_SUPERBUILD_CMAKE_VAR(OTB Shark_DIR)
 
 ADD_SUPERBUILD_CMAKE_VAR(OTB LIBSVM_INCLUDE_DIR)
 ADD_SUPERBUILD_CMAKE_VAR(OTB LIBSVM_LIBRARY)
@@ -181,7 +176,6 @@ ExternalProject_Add(OTB
       -DOTB_USE_MUPARSER:BOOL=${OTB_USE_MUPARSER}
       -DOTB_USE_MUPARSERX:BOOL=${OTB_USE_MUPARSERX}
       -DOTB_USE_OPENCV:BOOL=${OTB_USE_OPENCV}
-      -DOTB_USE_SHARK:BOOL=${OTB_USE_SHARK}
       -DOTB_USE_OPENJPEG:BOOL=${OTB_USE_OPENJPEG}
       -DOTB_USE_QT4:BOOL=${OTB_USE_QT4}
       -DOTB_USE_SIFTFAST:BOOL=${OTB_USE_SIFTFAST}
diff --git a/SuperBuild/CMake/External_shark.cmake b/SuperBuild/CMake/External_shark.cmake
deleted file mode 100644
index 1fdd685..0000000
--- a/SuperBuild/CMake/External_shark.cmake
+++ /dev/null
@@ -1,39 +0,0 @@
-if(NOT __EXTERNAL_SHARK__)
-set(__EXTERNAL_SHARK__ 1)
-
-if(USE_SYSTEM_SHARK)
-  message(STATUS "  Using SHARK system version")
-else()
-  SETUP_SUPERBUILD(PROJECT SHARK)
-  cmake_minimum_required(VERSION 3.1)
-  message(STATUS "  Using SHARK SuperBuild version")
-
-  # declare dependencies
-  ADDTO_DEPENDENCIES_IF_NOT_SYSTEM(SHARK BOOST)
-
-  ADD_SUPERBUILD_CMAKE_VAR(SHARK Boost_INCLUDE_DIR)
-  ADD_SUPERBUILD_CMAKE_VAR(SHARK Boost_LIBRARY_DIR)
-
-  ExternalProject_Add(SHARK
-    PREFIX SHARK
-    GIT_REPOSITORY "https://github.com/Shark-ML/Shark.git"
-    GIT_TAG "master"
-    SOURCE_DIR ${SHARK_SB_SRC}
-    BINARY_DIR ${SHARK_SB_BUILD_DIR}
-    INSTALL_DIR ${SB_INSTALL_PREFIX}
-    DOWNLOAD_DIR ${DOWNLOAD_LOCATION}
-    CMAKE_CACHE_ARGS
-      -DCMAKE_INSTALL_PREFIX:STRING=${SB_INSTALL_PREFIX}
-      -DCMAKE_PREFIX_PATH:STRING=${SB_INSTALL_PREFIX};${CMAKE_PREFIX_PATH}
-      -DCMAKE_BUILD_TYPE:STRING=Release
-      -DBUILD_SHARED_LIBS:BOOL=ON
-      -DBUILD_DOCS:BOOL=OFF
-      -DBUILD_EXAMPLES:BOOL=OFF
-      -DBUILD_TESTING:BOOL=OFF
-      -DENABLE_HDF5:BOOL=OFF
-      CMAKE_COMMAND ${SB_CMAKE_COMMAND}
-    )
-  SUPERBUILD_PATCH_SOURCE(SHARK "patch-for-at-rpath")
-  set(_SB_SHARK_DIR ${SB_INSTALL_PREFIX}/share/SHARK)
-endif()
-endif()
diff --git a/SuperBuild/CMakeLists.txt b/SuperBuild/CMakeLists.txt
index c997f75..fea7c77 100644
--- a/SuperBuild/CMakeLists.txt
+++ b/SuperBuild/CMakeLists.txt
@@ -182,7 +182,6 @@ SETUP_SYSTEM_LIBRARY(PROJECT QWT DEFAULT ${QT4_DEFAULT})
 SETUP_SYSTEM_LIBRARY(PROJECT GLEW DEFAULT ON)
 SETUP_SYSTEM_LIBRARY(PROJECT GLFW DEFAULT ON)
 SETUP_SYSTEM_LIBRARY(PROJECT GLUT DEFAULT ON)
-SETUP_SYSTEM_LIBRARY(PROJECT SHARK DEFAULT OFF)
 
 # Call OTB
 option(OTB_USE_6S "Enable module 6S in OTB" ON)
@@ -200,7 +199,6 @@ option(OTB_USE_OPENGL "Enable module OpenGL in OTB" OFF)
 option(OTB_USE_GLEW "Enable module GLEW in OTB" OFF)
 option(OTB_USE_GLFW "Enable module GLFW in OTB" OFF)
 option(OTB_USE_GLUT "Enable module GLUT in OTB" OFF)
-option(OTB_USE_SHARK "Enable module GLUT in OTB" OFF)
 
 if(${ENABLE_MONTEVERDI})
   set(OTB_USE_OPENGL ON)

A patch is needed to deal with HDF5 dependency of Shark:

diff --git a/SuperBuild/patches/SHARK/shark-1-disable-hdf5-linux.diff b/SuperBuild/patches/SHARK/shark-1-disable-hdf5-linux.diff
deleted file mode 100644
index d0b3df4..0000000
--- a/SuperBuild/patches/SHARK/shark-1-disable-hdf5-linux.diff
+++ /dev/null
@@ -1,258 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index df8be65..c6c3020 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -191,29 +191,32 @@ endif()
- #####################################################################
- #           HDF5 configuration
- #####################################################################
--find_package(HDF5 COMPONENTS C CXX HL QUIET)
--mark_as_advanced(HDF5_DIR)
--if(HDF5_FOUND)
--	if(HDF5_C_COMPILER_EXECUTABLE AND HDF5_CXX_COMPILER_EXECUTABLE)
--		message(STATUS "Checking HDF5 installation: HDF5 installation seems ok.")
--		include_directories( ${HDF5_INCLUDE_DIR} )
--		link_directories( ${HDF5_LIBRARY_DIR} )
--		list(APPEND LINK_LIBRARIES ${HDF5_LIBRARIES})
--	else()
--		message(STATUS "Checking HDF5 installation:HDF5 package might be broken.")
--		if(NOT( HDF5_C_COMPILER_EXECUTABLE))
--			message(STATUS "  C Compiler Extension not found.")
--		endif()
--		if(NOT( HDF5_CXX_COMPILER_EXECUTABLE))
--			message(STATUS "  CXX Compiler Extension not found.")
--		endif()
--			message(STATUS "Disabling HDF5.")
--		unset( HDF5_FOUND )
--	endif()
--else()
--	message(STATUS "HDF5 not found, skip")
-+option(ENABLE_HDF5 "Use HDF5" ON)
-+
-+if(ENABLE_HDF5)
-+  find_package(HDF5 COMPONENTS C CXX HL QUIET)
-+  mark_as_advanced(HDF5_DIR)
-+  if(HDF5_FOUND)
-+    if(HDF5_C_COMPILER_EXECUTABLE AND HDF5_CXX_COMPILER_EXECUTABLE)
-+      message(STATUS "Checking HDF5 installation: HDF5 installation seems ok.")
-+      include_directories( ${HDF5_INCLUDE_DIR} )
-+      link_directories( ${HDF5_LIBRARY_DIR} )
-+      list(APPEND LINK_LIBRARIES ${HDF5_LIBRARIES})
-+    else()
-+      message(STATUS "Checking HDF5 installation:HDF5 package might be broken.")
-+      if(NOT( HDF5_C_COMPILER_EXECUTABLE))
-+        message(STATUS "  C Compiler Extension not found.")
-+      endif()
-+      if(NOT( HDF5_CXX_COMPILER_EXECUTABLE))
-+        message(STATUS "  CXX Compiler Extension not found.")
-+      endif()
-+      message(STATUS "Disabling HDF5.")
-+      unset( HDF5_FOUND )
-+    endif()
-+  else()
-+    message(STATUS "HDF5 not found, skip")
-+  endif()
- endif()
--
Additions to the Learning/Supervised module

In order to propose a proof of concept for Shark integration, only the Random Forest algorithm is added. The following classes have been created:

Modules/Learning/Supervised/include/otbSharkRandomForestsMachineLearningModel.h
Modules/Learning/Supervised/include/otbSharkRandomForestsMachineLearningModel.txx
Modules/Learning/Supervised/include/otbRandomForestsMachineLearningModelFactory.h
Modules/Learning/Supervised/include/otbRandomForestsMachineLearningModelFactory.txx

A file containing utility functions for data conversion between Shark and OTB is also added:

Modules/Learning/Supervised/include/otbSharkUtils.h

Tests for the new classes are included:

a/Modules/Learning/Supervised/otb-module.cmake
Changes to the Learning/Supervised module

In order to overload the otbMachineLearningModel::PredictAll() method so that Shark can use batches, it has been made virtual:

diff --git a/Modules/Learning/Supervised/include/otbMachineLearningModel.h b/Modules/Learning/Supervised/include/otbMachineLearningModel.h
index 3b5cf87..b18e39e 100644
--- a/Modules/Learning/Supervised/include/otbMachineLearningModel.h
+++ b/Modules/Learning/Supervised/include/otbMachineLearningModel.h
@@ -103,7 +103,7 @@ public:
   virtual TargetSampleType Predict(const InputSampleType& input, ConfidenceValueType *quality = NULL) const = 0;
 
   /** Classify all samples in InputListSample and fill TargetListSample with the associated label */
-  virtual void PredictAll();
+  void PredictAll();
 
   /**\name Classification model file manipulation */
   //@{

Shark RF model is added to the factories:

diff --git a/Modules/Learning/Supervised/include/otbMachineLearningModelFactory.txx b/Modules/Learning/Supervised/include/otbMachineLearningModelFactory.txx
index f6be874..7fe0d43 100644
--- a/Modules/Learning/Supervised/include/otbMachineLearningModelFactory.txx
+++ b/Modules/Learning/Supervised/include/otbMachineLearningModelFactory.txx
@@ -31,9 +31,6 @@
 #ifdef OTB_USE_LIBSVM
 #include "otbLibSVMMachineLearningModelFactory.h"
 #endif
-#ifdef OTB_USE_SHARK
-#include "otbSharkRandomForestsMachineLearningModelFactory.h"
-#endif
 
 #include "itkMutexLockHolder.h"
 
@@ -108,9 +105,6 @@ MachineLearningModelFactory<TInputValue,TOutputValue>
   RegisterFactory(GradientBoostedTreeMachineLearningModelFactory<TInputValue,TOutputValue>::New());
   RegisterFactory(KNearestNeighborsMachineLearningModelFactory<TInputValue,TOutputValue>::New());
 #endif
-#ifdef OTB_USE_SHARK
-  RegisterFactory(SharkRandomForestsMachineLearningModelFactory<TInputValue,TOutputValue>::New());
-#endif
 
 }
Changes for the TrainImagesClassifier application

The following file is added to implement the parameter initialisation and training for the Shark Random Forest algorithm in the TrainImagesClassifierApplication:

Modules/Applications/AppClassification/include/otbTrainSharkRandomForests.txx

The needed changes to use Shark in the application are also applied.

Issues to be solved
Model factory and serialization

The SharkRFMachineModelFactory does not seem to work correctly, by I don't know if there is something on the Save/Load methods implementation or on the way Shark serializes the models.

Combining with ITK's multi-threading and Shark's use of OpenMP

In the prediction step, the otb::ImageClassificationFilter uses multi-threading, but the call to otb::MachineLearningModel::Predict() would use OpenMP on Shark's side. The use of the ITK_GLOBAL_NUMBER_OF_THREADS and OMP_NUM_THREADS may be a work around, but tests should be made.

JML: There is a way to avoid OpenMP calls before calling shark (the OMP_NUM_THREADS env var is a bit too invasive):

template <class TInputValue, class TOutputValue>
void
SharkRandomForestsMachineLearningModel<TInputValue,TOutputValue>
::PredictAll()
{
  unsigned int omp_num_threads = omp_get_num_threads();
  bool omp_dynamic = omp_get_dynamic();
  omp_set_num_threads(1);
  omp_set_dynamic(0);

  std::vector<shark::RealVector> features;
  Shark::ListSampleToSharkVector(this->GetInputListSample(), features);
  shark::Data<shark::RealVector> inputSamples = shark::createDataFromRange(features);
  shark::ArgMaxConverter<shark::RFClassifier> amc;
  amc.decisionFunction() = m_RFModel;
  auto prediction = amc(inputSamples);
  TargetListSampleType * targets = this->GetTargetListSample();
  targets->Clear();
  for(const auto& p : prediction.elements())
    {
    TargetSampleType target;
    target[0] = static_cast<TOutputValue>(p);
    targets->PushBack(target);
    }

  omp_set_num_threads(omp_num_threads);
  omp_set_dynamic(omp_dynamic);
}

However:

  • By default, shark does not do parallel prediction, because OpenMP is not active at OTB level. So no need for these guards unless you add -fopenmp flag
  • If you add -fopenmp flag, we probably need to do a proper FindOpenMP module, so that we get portable cmakedefine and can encapsulate the openmp code within #if guards
  • OpenMP can be enabled or not at shark level, so this is a bit hard to configure

JIA: There is an issue with ITK multithreading when using the PredictAll method. Since PredictAll uses state (the target samples to be filled by the prediction), the MachineLearningModel is not threadsafe. I have therefore taken the OpenMP route and the SharkImageClassificationFilter sets the number of threads to 1 if the batch mode is used (see below). Therefore, for parallel prediction, OTB has to be built with -fopenmp. This has also the benefit to allow parallel learning for Shark. I have been able to measure a significant increase both for training and classification using different values of the environment variable OMP_NUM_THREADS.

Using data batches: otb::MachineLearningModel::Predict()takes one sample at a time
    • should the otb::ImageClassificationFilter call a method using batches in ThreadedGenerateData()?
    • but how to keep the same interface for OpenCV: otb::MachineLearningModel::PredictAll() loops over samples and calls Predict for OpenCV?
    • make otb::MachineLearningModel::PredictAll() virtual and use batches for Shark?

JML: We absolutely need to change calls to PredictAll() in the ImageClassificationFilter, the Predict() is 100 times slower. I was thinking of making use of the ImageToListSampleAdaptor (if it works with VectorImage), which would prevent at least one deep-copy. To handle masking, we can create a MaskedImageToListSampleAdaptor (and add support for sub-region of image).

Regarding threading, we actually have two options here :

  1. Remove the ThreadedGenerateData() implementation in ImageClassificationFilter and make the default PredictAll() implementation in MachineLearningModel parallel using OpenMP. This would be perhaps more scalable, we can have a member at the MachineLearningModel level to tune how many threads are used for prediction, and it would benefit to any client of PredictAll() calls (instead of only the ImageClassificationFilter). On the other hand, pixel-wise operations filters in ITK/OTB are threaded library-wise with the ITK threading mechanism, so this would somehow break the rule.
  2. Keep ThreadedGenerateData() and ensure that Shark is not doing parallel predict.

JIA: I have implemented a SharkImageClassificationFilter which has a boolean flag allowing to choose between the single sample approach (Predict) and the batch approach (PredictAll). This flag is used in ThreadedGenerateData to dispatch to a BatchThreadedGenerateData or to a ClassicThreadedGenerateData. The flag is also used in BeforeGenerateData to set the number of threads to 1 if the batch mode is selected (see discussion about OpenMP above). The ClassicThreadedGenerateData is exactly the same implementation as the one in the ImageClassificationFilter (call to Predict() sample-wise). The batch mode has needed the addition of a ConfidenceListSample (and accessors), analogous to the TargetListSample to store the confidence (quality) values in the call to PredictAll(). This highlights the fact that all MachineLearningModels in OTB are unable to compute confidence in PredictAll.

ld.so is back

At least in my platform, when I run

for f in otbapp_* ; do echo "==== $f ====" ; nm $f | grep " u " | c++filt ; done

I get this

000000000033cd70 u guard variable for boost::serialization::singleton<boost::serialization::void_cast_detail::void_caster_primitive<boost_132::detail::sp_counted_base_impl<shark::blas::matrix<double, shark::blas::row_major>*, boost::serialization::null_deleter>, boost_132::detail::sp_counted_base> >::instance
000000000033cd58 u guard variable for boost::serialization::singleton<boost::serialization::void_cast_detail::void_caster_primitive<boost_132::detail::sp_counted_base_impl<shark::blas::vector<unsigned int>*, boost::serialization::null_deleter>, boost_132::detail::sp_counted_base> >::instance
000000000033ce40 u guard variable for boost::serialization::singleton<boost::serialization::extended_type_info_typeid<shark::RFClassifier> >::instance
000000000033ce28 u guard variable for boost::serialization::singleton<boost::serialization::extended_type_info_typeid<shark::UnlabeledData<shark::blas::vector<double> > > >::instance
000000000033d2b8 u guard variable for boost::serialization::singleton<boost::serialization::extended_type_info_typeid<shark::CARTClassifier<shark::blas::vector<double> >::NodeInfo> >::instance
000000000033d1d0 u guard variable for boost::serialization::singleton<boost::serialization::extended_type_info_typeid<shark::CARTClassifier<shark::blas::vector<double> > > >::instance
000000000033cdd0 u guard variable for boost::serialization::singleton<boost::serialization::extended_type_info_typeid<shark::blas::matrix<double, shark::blas::row_major> > >::instance
000000000033d2d0 u guard variable for boost::serialization::singleton<boost::serialization::extended_type_info_typeid<shark::blas::vector<double> > >::instance
Lots of warnings when compiling Shark with all flags on
  • shadowed vars
  • not used arguments
  • and non virtual overriden methods

I have started cleaning some things on a Shark clone here https://github.com/inglada/Shark and made the Superbuild point to this fork.

OTB seems to ignore #pragma omp for parallel loops in shark

OTB needs to be configured with CMAKE_CXX_FLAGS:STRING=-fopenmp

When will those changes be available (target release or date)?

It would be nice to have at least Shark Random Forests for 5.8

Who will be developing the proposed changes?

There is a feature branch here: https://git.orfeo-toolbox.org/otb.git/shortlog/refs/heads/shark-integration

Jordi has done the library and application code. Emmanuelle has done the ThidParty and Superbuild parts (because I was going to do this http://xkcd.com/1654/). The multi-threading and data batches issues seem now solved, but the following things have still to be done:

  1. Create a SharkMachineLearningModel class to factor common code for all algorithms (right now there is only a SharkRandomForestsMachineLearningModel which contains everything)
  2. Implement the CanReadFile for the model, so that Shark model format is correctly detected and can be used with a factory
  3. Add the Shark RF implementation to the training and classification OTB applications
    1. Needs the factory
    2. There will be issues with ld.so
  4. Implement regression

Community

Comments

N/A

Support

N/A

Corresponding Requests for Changes

Currently, none.