feat(hadoop): Add hdfs-utils

CHANGELOG.md

@@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file.
 - trino: Add version `438` ([#547]).
 - vector: Switch from version `0.33.0` to `0.35.0` ([#547], [#549]).
 - zookeeper: Add version `3.9.1` ([#551]).
+- hadoop: Add hdfs-utils ([#566]).
 
 ### Changed
 
@@ -89,6 +90,7 @@ All notable changes to this project will be documented in this file.
 [#563]: https://github.com/stackabletech/docker-images/pull/563
 [#564]: https://github.com/stackabletech/docker-images/pull/564
 [#565]: https://github.com/stackabletech/docker-images/pull/565
+[#566]: https://github.com/stackabletech/docker-images/pull/566
 [#567]: https://github.com/stackabletech/docker-images/pull/567
 
 ## [23.11.0] - 2023-11-30

conf.py

@@ -65,7 +65,7 @@
                 "product": "28.0.1",
                 # Java 17 should be fully supported as of 27.0.0 https://github.com/apache/druid/releases#27.0.0-highlights-java-17-support
                 # Did not work in a quick test due to reflection error:
-                # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class 
+                # Caused by: java.lang.reflect.InaccessibleObjectException: Unable to make protected final java.lang.Class
                 # java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int,java.security.ProtectionDomain) throws java.lang.ClassFormatError
                 "java-base": "11",
                 "jackson_dataformat_xml": "2.12.7", # from https://github.com/apache/druid/blob/b8201e31aa6b124049a61764309145baaad78db7/pom.xml#L100
@@ -84,6 +84,7 @@
                 "async_profiler": "2.9",
                 "jmx_exporter": "0.20.0",
                 "protobuf": "2.5.0",
+                # no HDFS utils, as e.g. patches from https://github.com/apache/hadoop/pull/6553 are missing. 3.2.x will probably removed in version 24.3 anyway.
                 "topology_provider": "0.2.0"
             },
             {
@@ -92,6 +93,7 @@
                 "async_profiler": "2.9",
                 "jmx_exporter": "0.20.0",
                 "protobuf": "3.7.1",
+                "hdfs_utils": "0.1.0",
                 "topology_provider": "0.2.0"
             },
             {
@@ -100,6 +102,7 @@
                 "async_profiler": "2.9",
                 "jmx_exporter": "0.20.0",
                 "protobuf": "3.7.1",
+                "hdfs_utils": "0.1.0",
                 "topology_provider": "0.2.0"
             },
         ],
@@ -346,8 +349,8 @@
             {
                 "product": "414",
                 "java-base": "17",
-                "opa_authorizer": "stackable0.2.0", 
-                "jmx_exporter": "0.20.0", 
+                "opa_authorizer": "stackable0.2.0",
+                "jmx_exporter": "0.20.0",
                 "storage_connector": "414"
             },
             {

hadoop/Dockerfile

@@ -6,6 +6,7 @@ ARG ASYNC_PROFILER
 ARG JMX_EXPORTER
 ARG PROTOBUF
 ARG TOPOLOGY_PROVIDER
+ARG HDFS_UTILS
 ARG TARGETARCH
 ARG TARGETOS
 
@@ -140,6 +141,11 @@ COPY --chown=stackable:stackable --from=builder /stackable/async-profiler /stack
 # N.B. the artifact name changed from 0.2.0 onwards i.e. from topology-provider-0.1.0.jar to hdfs-topology-provider-0.2.0.jar
 RUN curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-topology-provider/hdfs-topology-provider-${TOPOLOGY_PROVIDER}.jar -o /stackable/hadoop-${PRODUCT}/share/hadoop/tools/lib/hdfs-topology-provider-${TOPOLOGY_PROVIDER}.jar
 
+# The Stackable HDFS utils contain an OPA authorizer and group mapper
+RUN if [[ -n ${HDFS_UTILS} ]] ; then \
+        curl --fail -L -s -S https://repo.stackable.tech/repository/packages/hdfs-utils/hdfs-utils-${HDFS_UTILS}.jar -o /stackable/hadoop-${PRODUCT}/share/hadoop/tools/lib/hdfs-utils-${HDFS_UTILS}.jar ; \
+    fi
+
 RUN ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop
 COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin