s3, curvefs client, metasever: add s3 log prefix & refresh inode only…

… refresh s3 info

conf/s3.conf

@@ -18,6 +18,7 @@ s3.connect_timeout=60000
 s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
 s3.loglevel=4
+s3.logPrefix=/data/log/curve/aws_
 s3.async_thread_num=64
 # throttle
 s3.throttle.iopsTotalLimit=5000

curve-ansible/roles/generate_config/defaults/main.yml

@@ -250,6 +250,7 @@ s3_max_connections: 32
 s3_connect_timeout: 60000
 s3_request_timeout: 10000
 s3_loglevel: 4
+s3_logPrefix: /data/log/curve/aws_
 s3_async_thread_num: 64
 s3_throttle_iopsTotalLimit: 5000
 s3_throttle_iopsReadLimit: 5000

curve-ansible/roles/generate_config/templates/s3.conf.j2

@@ -16,6 +16,7 @@ s3.connect_timeout={{ s3_connect_timeout }}
 s3.request_timeout={{ s3_request_timeout }}
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
 s3.loglevel={{ s3_loglevel }}
+s3.logPrefix={{ s3_logPrefix }}
 s3.async_thread_num={{ s3_async_thread_num }}
 # throttle
 s3.throttle.iopsTotalLimit={{ s3_throttle_iopsTotalLimit }}

curvefs/conf/client.conf

@@ -111,6 +111,7 @@ s3.connect_timeout=60000
 s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
 s3.loglevel=4
+s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
 s3.async_thread_num=30
 # limit all inflight async requests' bytes, |0| means not limited
 s3.max_async_request_inflight_bytes=104857600

curvefs/conf/metaserver.conf

@@ -23,6 +23,7 @@ s3.connect_timeout=60000
 s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
 s3.loglevel=4
+s3.logPrefix=/tmp/curvefs/metaserver/aws_
 s3.async_thread_num=10
 # throttle
 s3.throttle.iopsTotalLimit=0

curvefs/proto/metaserver.proto

@@ -313,13 +313,13 @@ message GetOrModifyS3ChunkInfoRequest {
     required uint64 inodeId = 5;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 6;
     map<uint64, S3ChunkInfoList> s3ChunkInfoRemove = 7;
-    required bool returnInode = 8;
+    required bool returnS3ChunkInfoMap = 8;
 }
 
 message GetOrModifyS3ChunkInfoResponse {
     required MetaStatusCode statusCode = 1;
     optional uint64 appliedIndex = 2;
-    optional Inode inode = 3;
+    map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 3;
 }
 
 service MetaServerService {

curvefs/src/client/inode_wrapper.cpp

@@ -94,29 +94,19 @@ CURVEFS_ERROR InodeWrapper::Sync() {
 }
 
 CURVEFS_ERROR InodeWrapper::Refresh() {
-    if (dirty_) {
-        MetaStatusCode ret = metaClient_->UpdateInode(inode_);
-
-        if (ret != MetaStatusCode::OK) {
-            LOG(ERROR) << "metaClient_ UpdateInode failed, "
-                       << "MetaStatusCode: " << ret
-                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
-                       << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
-        }
-        dirty_ = false;
-    }
-    Inode newInode;
+    google::protobuf::Map<
+                uint64_t, S3ChunkInfoList> s3ChunkInfoMap;
     MetaStatusCode ret = metaClient_->GetOrModifyS3ChunkInfo(
-        inode_.fsid(), inode_.inodeid(), s3ChunkInfoAdd_, true, &newInode);
+        inode_.fsid(), inode_.inodeid(), s3ChunkInfoAdd_,
+        true, &s3ChunkInfoMap);
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ GetOrModifyS3ChunkInfo failed, "
                    << "MetaStatusCode: " << ret
                    << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                    << ", inodeid: " << inode_.inodeid();
         return MetaStatusCodeToCurvefsErrCode(ret);
     }
-    inode_.Swap(&newInode);
+    inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
     s3ChunkInfoAdd_.clear();
     return CURVEFS_ERROR::OK;
 }

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -482,8 +482,9 @@ MetaStatusCode MetaServerClientImpl::GetOrModifyS3ChunkInfo(
     uint32_t fsId, uint64_t inodeId,
     const google::protobuf::Map<
         uint64_t, S3ChunkInfoList> &s3ChunkInfos,
-    bool returnInode,
-    Inode *out) {
+    bool returnS3ChunkInfoMap,
+    google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out) {
     auto task = RPCTask {
         metaserverClientMetric_.appendS3ChunkInfo.qps.count << 1;
 
@@ -494,7 +495,7 @@ MetaStatusCode MetaServerClientImpl::GetOrModifyS3ChunkInfo(
         request.set_partitionid(partitionID);
         request.set_fsid(fsId);
         request.set_inodeid(inodeId);
-        request.set_returninode(returnInode);
+        request.set_returns3chunkinfomap(returnS3ChunkInfoMap);
         *(request.mutable_s3chunkinfoadd()) = s3ChunkInfos;
 
         curvefs::metaserver::MetaServerService_Stub stub(channel);
@@ -516,12 +517,12 @@ MetaStatusCode MetaServerClientImpl::GetOrModifyS3ChunkInfo(
                          << ", errorcode: " << ret
                          << ", errmsg: " << MetaStatusCode_Name(ret);
             return ret;
-        } else if (response.has_appliedindex() &&
-            (!returnInode || response.has_inode())) {
+        } else if (response.has_appliedindex()) {
             metaCache_->UpdateApplyIndex(CopysetGroupID(poolID, copysetID),
                                          response.appliedindex());
-            if (returnInode) {
-                out->CopyFrom(response.inode());
+            if (returnS3ChunkInfoMap) {
+                CHECK(out != nullptr) << "out ptr should be set.";
+                out->swap(*response.mutable_s3chunkinfomap());
             }
         } else {
             LOG(WARNING) << "GetOrModifyS3ChunkInfo,  inodeId: " << inodeId

curvefs/src/client/rpcclient/metaserver_client.h

@@ -86,8 +86,9 @@ class MetaServerClient {
         uint32_t fsId, uint64_t inodeId,
         const google::protobuf::Map<
             uint64_t, S3ChunkInfoList> &s3ChunkInfos,
-        bool returnInode = false,
-        Inode *out = nullptr) = 0;
+        bool returnS3ChunkInfoMap = false,
+        google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out = nullptr) = 0;
 
     virtual MetaStatusCode CreateInode(const InodeParam &param, Inode *out) = 0;
 
@@ -131,8 +132,9 @@ class MetaServerClientImpl : public MetaServerClient {
         uint32_t fsId, uint64_t inodeId,
         const google::protobuf::Map<
             uint64_t, S3ChunkInfoList> &s3ChunkInfos,
-        bool returnInode = false,
-        Inode *out = nullptr) override;
+        bool returnS3ChunkInfoMap = false,
+        google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out = nullptr) override;
 
     MetaStatusCode CreateInode(const InodeParam &param, Inode *out) override;
 

curvefs/src/metaserver/inode_manager.cpp

@@ -277,8 +277,9 @@ MetaStatusCode InodeManager::GetOrModifyS3ChunkInfo(
     uint32_t fsId, uint64_t inodeId,
     const google::protobuf::Map<uint64_t, S3ChunkInfoList> &s3ChunkInfoAdd,
     const google::protobuf::Map<uint64_t, S3ChunkInfoList> &s3ChunkInfoRemove,
-    bool returnInode,
-    Inode *out) {
+    bool returnS3ChunkInfoMap,
+    google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out) {
     VLOG(1) << "GetOrModifyS3ChunkInfo, fsId: " << fsId
             << ", inodeId: " << inodeId;
 
@@ -343,14 +344,13 @@ MetaStatusCode InodeManager::GetOrModifyS3ChunkInfo(
             }
         }
     }
-
-    if (returnInode) {
-        out->Swap(&old);
+    if (returnS3ChunkInfoMap) {
+        out->swap(*(old.mutable_s3chunkinfomap()));
     }
 
     VLOG(1) << "GetOrModifyS3ChunkInfo success, fsId: " << fsId
-            << ", inodeId: " << inodeId
-            << ", inodesize: " << old.ByteSizeLong();
+            << ", inodeId: " << inodeId;
+
     return MetaStatusCode::OK;
 }
 

curvefs/src/metaserver/inode_manager.h

@@ -59,8 +59,9 @@ class InodeManager {
         const google::protobuf::Map<uint64_t, S3ChunkInfoList> &s3ChunkInfoAdd,
         const google::protobuf::Map<uint64_t, S3ChunkInfoList>
             &s3ChunkInfoRemove,
-        bool returnInode,
-        Inode *out);
+        bool returnS3ChunkInfoMap,
+        google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out);
 
     MetaStatusCode UpdateInodeWhenCreateOrRemoveSubNode(uint32_t fsId,
         uint64_t inodeId, bool isCreate);

curvefs/src/metaserver/metastore.cpp

@@ -580,14 +580,10 @@ MetaStatusCode MetaStoreImpl::GetOrModifyS3ChunkInfo(
         response->set_statuscode(status);
         return status;
     }
-    Inode out;
     MetaStatusCode status = partition->GetOrModifyS3ChunkInfo(fsId, inodeId,
         request->s3chunkinfoadd(), request->s3chunkinforemove(),
-        request->returninode(), &out);
+        request->returns3chunkinfomap(), response->mutable_s3chunkinfomap());
     response->set_statuscode(status);
-    if (request->returninode()) {
-        response->mutable_inode()->Swap(&out);
-    }
     return status;
 }
 

curvefs/src/metaserver/partition.cpp

@@ -248,8 +248,9 @@ MetaStatusCode Partition::GetOrModifyS3ChunkInfo(
     const google::protobuf::Map<uint64_t, S3ChunkInfoList> &s3ChunkInfoAdd,
     const google::protobuf::Map<uint64_t, S3ChunkInfoList>
         &s3ChunkInfoRemove,
-    bool returnInode,
-    Inode *out) {
+    bool returnS3ChunkInfoMap,
+    google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out) {
     if (!IsInodeBelongs(fsId, inodeId)) {
         return MetaStatusCode::PARTITION_ID_MISSMATCH;
     }
@@ -259,7 +260,8 @@ MetaStatusCode Partition::GetOrModifyS3ChunkInfo(
     }
 
     return inodeManager_->GetOrModifyS3ChunkInfo(
-        fsId, inodeId, s3ChunkInfoAdd, s3ChunkInfoRemove, returnInode, out);
+        fsId, inodeId, s3ChunkInfoAdd, s3ChunkInfoRemove,
+        returnS3ChunkInfoMap, out);
 }
 
 MetaStatusCode Partition::InsertInode(const Inode& inode) {

curvefs/src/metaserver/partition.h

@@ -83,8 +83,9 @@ class Partition {
         const google::protobuf::Map<uint64_t, S3ChunkInfoList> &s3ChunkInfoAdd,
         const google::protobuf::Map<uint64_t, S3ChunkInfoList>
             &s3ChunkInfoRemove,
-        bool returnInode,
-        Inode *out);
+        bool returnS3ChunkInfoMap,
+        google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out);
 
     MetaStatusCode InsertInode(const Inode& inode);
 

curvefs/src/metaserver/s3compact_manager.cpp

@@ -102,6 +102,7 @@ void S3CompactWorkQueueOption::Init(std::shared_ptr<Configuration> conf) {
     s3opts.s3Address = "";
     s3opts.bucketName = "";
     conf->GetValueFatalIfFail("s3.loglevel", &s3opts.loglevel);
+    conf->GetStringValue("s3.logPrefix", &s3opts.logPrefix);
     conf->GetValueFatalIfFail("s3.http_scheme", &s3opts.scheme);
     conf->GetValueFatalIfFail("s3.verify_SSL", &s3opts.verifySsl);
     conf->GetValueFatalIfFail("s3.max_connections", &s3opts.maxConnections);

curvefs/src/metaserver/s3compact_wq_impl.cpp

@@ -307,7 +307,7 @@ MetaStatusCode S3CompactWorkQueueImpl::UpdateInode(
     request.set_inodeid(inodeId);
     *request.mutable_s3chunkinfoadd() = std::move(s3ChunkInfoAdd);
     *request.mutable_s3chunkinforemove() = std::move(s3ChunkInfoRemove);
-    request.set_returninode(false);
+    request.set_returns3chunkinfomap(false);
     GetOrModifyS3ChunkInfoResponse response;
     S3CompactWorkQueueImpl::GetOrModifyS3ChunkInfoClosure done;
     // if copysetnode change to nullptr, maybe crash

curvefs/test/client/mock_metaserver_client.h

@@ -80,8 +80,9 @@ class MockMetaServerClient : public MetaServerClient {
         uint32_t fsId, uint64_t inodeId,
         const google::protobuf::Map<
             uint64_t, S3ChunkInfoList> &s3ChunkInfos,
-        bool returnInode,
-        Inode *out));
+        bool returnS3ChunkInfoMap,
+        google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> *out));
 
     MOCK_METHOD2(CreateInode, MetaStatusCode(
             const InodeParam &param, Inode *out));

curvefs/test/metaserver/inode_manager_test.cpp

@@ -169,45 +169,49 @@ TEST_F(InodeManagerTest, test1) {
         s3ChunkInfoAdd[j] = list[j];
     }
 
-    Inode inode3Out;
+    google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> s3Out1;
     ASSERT_EQ(MetaStatusCode::OK,
         manager.GetOrModifyS3ChunkInfo(
             fsId, inode3.inodeid(), s3ChunkInfoAdd, s3ChunkInfoRemove,
-            true, &inode3Out));
+            true, &s3Out1));
 
-    ASSERT_EQ(10, inode3Out.s3chunkinfomap_size());
+    ASSERT_EQ(10, s3Out1.size());
     for (int j = 0; j < 10; j++) {
         ASSERT_TRUE(MessageDifferencer::Equals(s3ChunkInfoAdd[j],
-                inode3Out.s3chunkinfomap().at(j)));
+                s3Out1.at(j)));
     }
 
     // Idempotent test
-    Inode inode4Out;
+    google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> s3Out2;
     ASSERT_EQ(MetaStatusCode::OK,
         manager.GetOrModifyS3ChunkInfo(
             fsId, inode3.inodeid(), s3ChunkInfoAdd, s3ChunkInfoRemove,
-            true, &inode4Out));
+            true, &s3Out2));
 
-    ASSERT_EQ(10, inode4Out.s3chunkinfomap_size());
+    ASSERT_EQ(10, s3Out2.size());
     for (int j = 0; j < 10; j++) {
         ASSERT_TRUE(MessageDifferencer::Equals(s3ChunkInfoAdd[j],
-                inode4Out.s3chunkinfomap().at(j)));
+                s3Out2.at(j)));
     }
 
-    Inode inode5Out;
+    google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> s3Out3;
     ASSERT_EQ(MetaStatusCode::OK,
         manager.GetOrModifyS3ChunkInfo(
             fsId, inode3.inodeid(), s3ChunkInfoRemove, s3ChunkInfoAdd,
-        true, &inode5Out));
-    ASSERT_EQ(0, inode5Out.s3chunkinfomap_size());
+        true, &s3Out3));
+    ASSERT_EQ(0, s3Out3.size());
 
     // Idempotent test
-    Inode inode6Out;
+    google::protobuf::Map<
+            uint64_t, S3ChunkInfoList> s3Out4;
     ASSERT_EQ(MetaStatusCode::OK,
         manager.GetOrModifyS3ChunkInfo(
             fsId, inode3.inodeid(), s3ChunkInfoRemove, s3ChunkInfoAdd,
-            true, &inode6Out));
-    ASSERT_EQ(0, inode6Out.s3chunkinfomap_size());
+            true, &s3Out4));
+    ASSERT_EQ(0, s3Out4.size());
 }
 }  // namespace metaserver
 }  // namespace curvefs

src/common/s3_adapter.cpp

@@ -71,6 +71,7 @@ Aws::String GetObjectRequestRange(uint64_t offset, uint64_t len) {
 void InitS3AdaptorOption(Configuration *conf,
     S3AdapterOption *s3Opt) {
     LOG_IF(FATAL, !conf->GetIntValue("s3.loglevel", &s3Opt->loglevel));
+    LOG_IF(FATAL, !conf->GetStringValue("s3.logPrefix", &s3Opt->logPrefix));
     LOG_IF(FATAL, !conf->GetStringValue("s3.endpoint", &s3Opt->s3Address));
     LOG_IF(FATAL, !conf->GetStringValue("s3.ak", &s3Opt->ak));
     LOG_IF(FATAL, !conf->GetStringValue("s3.sk", &s3Opt->sk));
@@ -120,6 +121,8 @@ void S3Adapter::Init(const S3AdapterOption &option) {
     auto initSDK = [&]() {
         AWS_SDK_OPTIONS.loggingOptions.logLevel =
             Aws::Utils::Logging::LogLevel(option.loglevel);
+        AWS_SDK_OPTIONS.loggingOptions.defaultLogPrefix =
+            option.logPrefix.c_str();
         Aws::InitAPI(AWS_SDK_OPTIONS);
     };
     std::call_once(S3INIT_FLAG, initSDK);

src/common/s3_adapter.h

@@ -71,6 +71,7 @@ struct S3AdapterOption {
     std::string s3Address;
     std::string bucketName;
     int loglevel;
+    std::string logPrefix;
     int scheme;
     bool verifySsl;
     int maxConnections;