HDDS-10655. Support PutObjectTagging, GetObjectTagging, and DeleteObj…

…ectTagging (apache#6756)
aswinshakil · Nov 17, 2024 · 3e278b7 · 3e278b7
1 parent 036e727
commit 3e278b7
Show file tree

Hide file tree

Showing 75 changed files with 4,043 additions and 138 deletions.
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
@@ -47,6 +47,9 @@ public enum OzoneManagerVersion implements ComponentVersion {
   LIGHTWEIGHT_LIST_STATUS(8, "OzoneManager version that supports lightweight"
       + " listStatus API."),
 
+  S3_OBJECT_TAGGING_API(9, "OzoneManager version that supports S3 object tagging APIs, such as " +
+      "PutObjectTagging, GetObjectTagging, and DeleteObjectTagging"),
+
   FUTURE_VERSION(-1, "Used internally in the client when the server side is "
       + " newer and an unknown server version has arrived to the client.");
 

diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -1046,6 +1046,37 @@ public void setTimes(String keyName, long mtime, long atime)
     proxy.setTimes(ozoneObj, keyName, mtime, atime);
   }
 
+  /**
+   * Gets the tags for an existing key.
+   * @param keyName Key name.
+   * @return Tags for the specified key.
+   * @throws IOException
+   */
+  public Map<String, String> getObjectTagging(String keyName)
+      throws IOException {
+    return proxy.getObjectTagging(volumeName, name, keyName);
+  }
+
+  /**
+   * Sets the tags to an existing key.
+   * @param keyName Key name.
+   * @param tags Tags to set on the key.
+   * @throws IOException
+   */
+  public void putObjectTagging(String keyName, Map<String, String> tags)
+      throws IOException {
+    proxy.putObjectTagging(volumeName, name, keyName, tags);
+  }
+
+  /**
+   * Removes all the tags from an existing key.
+   * @param keyName Key name
+   * @throws IOException
+   */
+  public void deleteObjectTagging(String keyName) throws IOException {
+    proxy.deleteObjectTagging(volumeName, name, keyName);
+  }
+
   public void setSourcePathExist(boolean b) {
     this.sourcePathExist = b;
   }

diff --git a/...op-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/...op-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -1311,4 +1311,38 @@ void setTimes(OzoneObj obj, String keyName, long mtime, long atime)
    * @throws IOException
    */
   void recoverKey(OmKeyArgs args, long clientID) throws IOException;
+
+  /**
+   * Gets the tags for an existing key.
+   * @param volumeName Volume name.
+   * @param bucketName Bucket name.
+   * @param keyName Key name.
+   * @return tags for the specified key.
+   * @throws IOException
+   */
+  Map<String, String> getObjectTagging(String volumeName, String bucketName, String keyName)
+      throws IOException;
+
+  /**
+   * Sets the tags to an existing key.
+   * @param volumeName Volume name.
+   * @param bucketName Bucket name.
+   * @param keyName Key name.
+   * @param tags Tags to set on the key.
+   * @throws IOException
+   */
+  void putObjectTagging(String volumeName, String bucketName, String keyName,
+                        Map<String, String> tags) throws IOException;
+
+
+  /**
+   * Removes all the tags from the specified key.
+   * @param volumeName Volume name.
+   * @param bucketName Bucket name.
+   * @param keyName Key name.
+   * @throws IOException
+   */
+  void deleteObjectTagging(String volumeName, String bucketName, String keyName)
+      throws IOException;
+
 }
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -2727,6 +2727,61 @@ public void recoverKey(OmKeyArgs args, long clientID) throws IOException {
     ozoneManagerClient.recoverKey(args, clientID);
   }
 
+  @Override
+  public Map<String, String> getObjectTagging(String volumeName, String bucketName, String keyName)
+      throws IOException {
+    if (omVersion.compareTo(OzoneManagerVersion.S3_OBJECT_TAGGING_API) < 0) {
+      throw new IOException("OzoneManager does not support S3 object tagging API");
+    }
+
+    verifyVolumeName(volumeName);
+    verifyBucketName(bucketName);
+    Preconditions.checkNotNull(keyName);
+    OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setKeyName(keyName)
+        .build();
+    return ozoneManagerClient.getObjectTagging(keyArgs);
+  }
+
+  @Override
+  public void putObjectTagging(String volumeName, String bucketName,
+                               String keyName, Map<String, String> tags) throws IOException {
+    if (omVersion.compareTo(OzoneManagerVersion.S3_OBJECT_TAGGING_API) < 0) {
+      throw new IOException("OzoneManager does not support S3 object tagging API");
+    }
+
+    verifyVolumeName(volumeName);
+    verifyBucketName(bucketName);
+    Preconditions.checkNotNull(keyName);
+    OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setKeyName(keyName)
+        .addAllTags(tags)
+        .build();
+    ozoneManagerClient.putObjectTagging(keyArgs);
+  }
+
+  @Override
+  public void deleteObjectTagging(String volumeName, String bucketName,
+                                  String keyName) throws IOException {
+    if (omVersion.compareTo(OzoneManagerVersion.S3_OBJECT_TAGGING_API) < 0) {
+      throw new IOException("OzoneManager does not support S3 object tagging API");
+    }
+
+    verifyVolumeName(volumeName);
+    verifyBucketName(bucketName);
+    Preconditions.checkNotNull(keyName);
+    OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+        .setVolumeName(volumeName)
+        .setBucketName(bucketName)
+        .setKeyName(keyName)
+        .build();
+    ozoneManagerClient.deleteObjectTagging(keyArgs);
+  }
+
   private static ExecutorService createThreadPoolExecutor(
        int corePoolSize, int maximumPoolSize, String threadNameFormat) {
     return new ThreadPoolExecutor(corePoolSize, maximumPoolSize,

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -278,6 +278,7 @@ public static boolean isReadOnly(
     case SetSafeMode:
     case PrintCompactionLogDag:
     case GetSnapshotInfo:
+    case GetObjectTagging:
     case GetQuotaRepairStatus:
     case StartQuotaRepair:
       return true;
@@ -339,6 +340,8 @@ public static boolean isReadOnly(
     case AbortExpiredMultiPartUploads:
     case SetSnapshotProperty:
     case QuotaRepair:
+    case PutObjectTagging:
+    case DeleteObjectTagging:
     case UnknownCommand:
       return false;
     case EchoRPC:

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
@@ -30,6 +30,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Protocol for OmMetadataReader's.
@@ -165,4 +166,11 @@ ListKeysLightResult listKeysLight(String volumeName, String bucketName,
    * @throws IOException if there is error.
    */
   List<OzoneAcl> getAcl(OzoneObj obj) throws IOException;
+
+  /**
+   * Gets the tags for the specified key.
+   * @param args Key args
+   * @return Tags associated with the key.
+   */
+  Map<String, String> getObjectTagging(OmKeyArgs args) throws IOException;
 }
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyValueUtil.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/KeyValueUtil.java
@@ -34,8 +34,8 @@ private KeyValueUtil() {
   /**
    * Parse Key,Value map data from protobuf representation.
    */
-  public static Map<String, String> getFromProtobuf(List<KeyValue> metadata) {
-    return metadata.stream()
+  public static Map<String, String> getFromProtobuf(List<KeyValue> keyValueList) {
+    return keyValueList.stream()
         .collect(Collectors.toMap(KeyValue::getKey,
             KeyValue::getValue));
   }

diff --git a/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/...-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -1145,6 +1145,31 @@ void setTimes(OmKeyArgs keyArgs, long mtime, long atime)
   boolean setSafeMode(SafeModeAction action, boolean isChecked)
       throws IOException;
 
+  /**
+   * Gets the tags for the specified key.
+   * @param args Key args
+   * @return Tags associated with the key.
+   */
+  Map<String, String> getObjectTagging(OmKeyArgs args) throws IOException;
+
+  /**
+   * Sets the tags to an existing key.
+   * @param args Key args
+   */
+  default void putObjectTagging(OmKeyArgs args) throws IOException {
+    throw new UnsupportedOperationException("OzoneManager does not require " +
+        "this to be implemented, as write requests use a new approach.");
+  }
+
+  /**
+   * Removes all the tags from the specified key.
+   * @param args Key args
+   */
+  default void deleteObjectTagging(OmKeyArgs args) throws IOException {
+    throw new UnsupportedOperationException("OzoneManager does not require " +
+        "this to be implemented, as write requests use a new approach.");
+  }
+
   /**
    * Get status of last triggered quota repair in OM.
    * @return String

diff --git a/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/...ava/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -108,6 +108,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyArgs;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeyRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteKeysRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteObjectTaggingRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteSnapshotRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteTenantRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteTenantResponse;
@@ -125,6 +126,8 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetFileStatusResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetKeyInfoResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetObjectTaggingRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetObjectTaggingResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3SecretResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextRequest;
@@ -174,6 +177,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrintCompactionLogDagRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PutObjectTaggingRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseRequest;
@@ -2580,6 +2584,72 @@ public void startQuotaRepair(List<String> buckets) throws IOException {
     handleError(submitRequest(omRequest));
   }
 
+  @Override
+  public Map<String, String> getObjectTagging(OmKeyArgs args) throws IOException {
+    KeyArgs keyArgs = KeyArgs.newBuilder()
+        .setVolumeName(args.getVolumeName())
+        .setBucketName(args.getBucketName())
+        .setKeyName(args.getKeyName())
+        .build();
+
+    GetObjectTaggingRequest req =
+        GetObjectTaggingRequest.newBuilder()
+            .setKeyArgs(keyArgs)
+            .build();
+
+    OMRequest omRequest = createOMRequest(Type.GetObjectTagging)
+        .setGetObjectTaggingRequest(req)
+        .build();
+
+    GetObjectTaggingResponse resp =
+        handleError(submitRequest(omRequest)).getGetObjectTaggingResponse();
+
+    return KeyValueUtil.getFromProtobuf(resp.getTagsList());
+  }
+
+  @Override
+  public void putObjectTagging(OmKeyArgs args) throws IOException {
+    KeyArgs keyArgs = KeyArgs.newBuilder()
+        .setVolumeName(args.getVolumeName())
+        .setBucketName(args.getBucketName())
+        .setKeyName(args.getKeyName())
+        .addAllTags(KeyValueUtil.toProtobuf(args.getTags()))
+        .build();
+
+    PutObjectTaggingRequest req =
+        PutObjectTaggingRequest.newBuilder()
+            .setKeyArgs(keyArgs)
+            .build();
+
+    OMRequest omRequest = createOMRequest(Type.PutObjectTagging)
+        .setPutObjectTaggingRequest(req)
+        .build();
+
+    OMResponse omResponse = submitRequest(omRequest);
+    handleError(omResponse);
+  }
+
+  @Override
+  public void deleteObjectTagging(OmKeyArgs args) throws IOException {
+    KeyArgs keyArgs = KeyArgs.newBuilder()
+        .setVolumeName(args.getVolumeName())
+        .setBucketName(args.getBucketName())
+        .setKeyName(args.getKeyName())
+        .build();
+
+    DeleteObjectTaggingRequest req =
+        DeleteObjectTaggingRequest.newBuilder()
+            .setKeyArgs(keyArgs)
+            .build();
+
+    OMRequest omRequest = createOMRequest(Type.DeleteObjectTagging)
+        .setDeleteObjectTaggingRequest(req)
+        .build();
+
+    OMResponse omResponse = submitRequest(omRequest);
+    handleError(omResponse);
+  }
+
   private SafeMode toProtoBuf(SafeModeAction action) {
     switch (action) {
     case ENTER:

diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -44,6 +44,7 @@ Put object to s3
 Get object from s3
     ${result} =                 Execute AWSS3ApiCli        get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 /tmp/testfile.result
     Compare files               /tmp/testfile              /tmp/testfile.result
+                                Should not contain        ${result}   TagCount
     ${result} =                 Execute AWSS3ApiCli        get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte /tmp/zerobyte.result
     Compare files               /tmp/zerobyte              /tmp/zerobyte.result