[Commits] 5349b6d4: Import the patch from https://github.com/rockset/rocksdb-cloud/issues/45

Sergei Petrunia psergey at askmonty.org
Wed Jan 30 16:27:07 EET 2019


revision-id: 5349b6d4a81fbc4661c83d722e55159ee9d39ee1 ()
parent(s): d5b69eb25461da5ff82bf0598aeb5f7f85362bb1
author: Sergei Petrunia
committer: Sergei Petrunia
timestamp: 2019-01-30 17:27:07 +0300
message:

Import the patch from https://github.com/rockset/rocksdb-cloud/issues/45

Will need further adjustments.

---
 CMakeLists.txt                            | 55 ++++++++++++++++++++++++++++++-
 cloud/aws/aws_env.cc                      | 53 ++++++++++++++++++++++++++---
 include/rocksdb/cloud/cloud_env_options.h | 44 +++++++++++++++++++++++++
 3 files changed, 147 insertions(+), 5 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b1fdf913..df5db6bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -49,6 +49,47 @@ option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
 option(WITH_ZLIB "build with zlib" OFF)
 option(WITH_ZSTD "build with zstd" OFF)
+
+
+#    find_package(snappy REQUIRED)
+#    add_definitions(-DSNAPPY)
+#    include_directories(${SNAPPY_INCLUDE_DIR})
+#    list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
+
+#find_package(rdkafka REQUIRED)
+#add_definitions(-DUSE_KAFKA)
+#include_directories(${RDKAFKA_INCLUDE_DIR})
+#list(APPEND THIRDPARTY_LIBS ${RDKAFKA_LIBRARIES})
+#CMAKE_LINK_LIBRARY_FLAG
+# set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lrdkafka++")
+
+# [MS] Adding debug info
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -g")
+
+# [MS] Adding librdkafka++
+# rdkafka uses pkgconfig instead of Findrdkafka.cmake
+# https://stackoverflow.com/questions/29191855/what-is-the-proper-way-to-use-pkg-config-from-cmake
+add_definitions(-DUSE_KAFKA)
+# Using system installed rdkafka
+# find_package(PkgConfig REQUIRED)
+# pkg_check_modules(RDKAFKA REQUIRED rdkafka++)
+# include_directories(${RDKAFKA_INCLUDE_DIR})
+# list(APPEND THIRDPARTY_LIBS ${RDKAFKA_LIBRARIES})
+# use custom location for rdkafka
+include_directories(/home/myrocks/Work/librdkafka/librdkafka-0.11.5/install/usr/local/include)
+list(APPEND THIRDPARTY_LIBS rdkafka rdkafka++)
+link_directories(/home/myrocks/Work/librdkafka/librdkafka-0.11.5/install/usr/local/lib)
+
+# [MS] Adding AWS libs
+add_definitions(-DUSE_AWS)
+include_directories(/home/myrocks/Work/aws/aws-sdk-cpp_master_2018-06-25/install/usr/local/include)
+# -laws-cpp-sdk-s3 -laws-cpp-sdk-kinesis -laws-cpp-sdk-core
+list(APPEND THIRDPARTY_LIBS aws-cpp-sdk-s3 aws-cpp-sdk-kinesis aws-cpp-sdk-core)
+link_directories(/home/myrocks/Work/aws/aws-sdk-cpp_master_2018-06-25/install/usr/local/lib)
+
+
 if(MSVC)
   # Defaults currently different for GFLAGS.
   #  We will address find_package work a little later
@@ -677,6 +718,18 @@ set(SOURCES
         utilities/ttl/db_ttl_impl.cc
         utilities/write_batch_with_index/write_batch_with_index.cc
         utilities/write_batch_with_index/write_batch_with_index_internal.cc
+  cloud/aws/aws_env.cc                                          
+  cloud/aws/aws_kafka.cc                                        
+  cloud/aws/aws_kinesis.cc                                      
+  cloud/aws/aws_log.cc                                          
+  cloud/aws/aws_retry.cc                                        
+  cloud/aws/aws_s3.cc                                           
+  cloud/db_cloud_impl.cc                                        
+  cloud/cloud_env.cc                                            
+  cloud/cloud_env_options.cc                                    
+  cloud/manifest_reader.cc                                      
+  cloud/purge.cc                                                
+  cloud/cloud_manifest.cc                                       
         $<TARGET_OBJECTS:build_version>)
 
 if(HAVE_SSE42 AND NOT FORCE_SSE42)
@@ -1026,7 +1079,7 @@ if(WITH_TESTS)
         )
 
   # Tests are excluded from Release builds
-  set(TEST_EXES ${TESTS})
+set(TEST_EXES ${TESTS})
 
   foreach(sourcefile ${TEST_EXES})
       get_filename_component(exename ${sourcefile} NAME_WE)
diff --git a/cloud/aws/aws_env.cc b/cloud/aws/aws_env.cc
index 7290561d..1c3bf19e 100644
--- a/cloud/aws/aws_env.cc
+++ b/cloud/aws/aws_env.cc
@@ -278,11 +278,56 @@ AwsEnv::AwsEnv(Env* underlying_env, const std::string& src_bucket_prefix,
          creds ? "[given]" : "[not given]");
 
   base_env_ = underlying_env;
-  Aws::InitAPI(Aws::SDKOptions());
-  // create AWS S3 client with appropriate timeouts
+
+  // Get additional data for 'sdk_options' and 'config' from cloud_env_options.aws_options
+  // Check if the corresponded value is not usable - use defaults
+
+  Aws::SDKOptions sdk_options;
   Aws::Client::ClientConfiguration config;
-  config.connectTimeoutMs = 30000;
-  config.requestTimeoutMs = 600000;
+
+  // General SDK options: logging
+  if (cloud_env_options.aws_options.logLevel != -1)
+    sdk_options.loggingOptions.logLevel = (Aws::Utils::Logging::LogLevel)cloud_env_options.aws_options.logLevel;
+  Aws::InitAPI(sdk_options);
+
+  // Timeout control
+  if (cloud_env_options.aws_options.connectTimeoutMs != -1)
+    config.connectTimeoutMs = cloud_env_options.aws_options.connectTimeoutMs;
+  else
+    config.connectTimeoutMs = 30000;
+
+  if (cloud_env_options.aws_options.requestTimeoutMs != -1)
+    config.requestTimeoutMs = cloud_env_options.aws_options.requestTimeoutMs;
+  else
+    config.requestTimeoutMs = 600000;
+
+  // Endpoint override
+  if (cloud_env_options.aws_options.endpointOverride != "")
+    config.endpointOverride = cloud_env_options.aws_options.endpointOverride.c_str();
+
+  if (cloud_env_options.aws_options.scheme != "")
+    config.scheme = Aws::Http::SchemeMapper::FromString(cloud_env_options.aws_options.scheme.c_str());
+
+  if (cloud_env_options.aws_options.verifySSL != -1)
+    config.verifySSL = (bool)cloud_env_options.aws_options.verifySSL;
+
+  // Proxy settings, not using by default and if proxyHost == ""
+  if (cloud_env_options.aws_options.proxyHost != "")
+  {
+    config.proxyHost = cloud_env_options.aws_options.proxyHost.c_str();
+
+    if (cloud_env_options.aws_options.proxyScheme != "")
+      config.proxyScheme = Aws::Http::SchemeMapper::FromString(cloud_env_options.aws_options.proxyScheme.c_str());
+
+    if (cloud_env_options.aws_options.proxyPort > 0)
+      config.proxyPort = cloud_env_options.aws_options.proxyPort;
+
+    if (cloud_env_options.aws_options.proxyUserName != "")
+      config.proxyUserName = cloud_env_options.aws_options.proxyUserName.c_str();
+
+    if (cloud_env_options.aws_options.proxyPassword != "")
+      config.proxyPassword = cloud_env_options.aws_options.proxyPassword.c_str();
+  }
 
   // Setup how retries need to be done
   config.retryStrategy =
diff --git a/include/rocksdb/cloud/cloud_env_options.h b/include/rocksdb/cloud/cloud_env_options.h
index af7278fd..57a9bfed 100644
--- a/include/rocksdb/cloud/cloud_env_options.h
+++ b/include/rocksdb/cloud/cloud_env_options.h
@@ -46,6 +46,47 @@ class KafkaLogOptions {
    std::unordered_map<std::string, std::string> client_config_params;
 };
 
+// Extra data to fill-in Aws::SDKOptions and Aws::Client::ClientConfiguration
+// inside AwsEnv::AwsEnv()
+// Not usable values, using defaults instead:
+//  -1 for int/long
+//  "" for std::string
+class AwsOptions {
+ public:
+  // General SDK options: logging
+  int logLevel;                    // see https://sdk.amazonaws.com/cpp/api/0.14.3/aws-cpp-sdk-core_2include_2aws_2core_2utils_2logging_2_log_level_8h_source.html
+
+  // Timeout control
+  long requestTimeoutMs;
+  long connectTimeoutMs;
+
+  // Endpoint override
+  std::string endpointOverride;    // local S3/Minio server "127.0.0.1:9000"
+  std::string scheme;              // "http" or "https"
+  int verifySSL;                   // 0 - don't verify, 1 - do verify
+
+  // Proxy settings, not using by default and if proxyHost == ""
+  std::string proxyScheme;
+  std::string proxyHost;
+  unsigned proxyPort;
+  std::string proxyUserName;
+  std::string proxyPassword;
+
+  // Setting not usable values as defaults (librocksdb/AWS defaults will be used)
+  AwsOptions() :
+    logLevel(-1),
+    requestTimeoutMs(-1),
+    connectTimeoutMs(-1),
+    endpointOverride(""),
+    scheme(""),
+    verifySSL(-1),
+    proxyScheme(""),
+    proxyHost(""),
+    proxyPort(0),
+    proxyUserName(""),
+    proxyPassword("") {}
+};
+
 enum class CloudRequestOpType {
   kReadOp,
   kWriteOp,
@@ -77,6 +118,9 @@ class CloudEnvOptions {
   // Only used if keep_local_log_files is true and log_type is kKafka.
   KafkaLogOptions kafka_log_options;
 
+  // To specify AWS related extras: server IP:port, scheme, timeouts, proxy.
+  AwsOptions aws_options;
+
   //
   // If true,  then sst files are stored locally and uploaded to the cloud in
   // the background. On restart, all files from the cloud that are not present


More information about the commits mailing list