Azure Blob Storage Support 1311

fluss-filesystems/fluss-fs-azure/pom.xml

@@ -0,0 +1,234 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.fluss</groupId>
+        <artifactId>fluss-filesystems</artifactId>
+        <version>0.9-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>fluss-fs-azure</artifactId>
+    <name>Fluss : FileSystems : Azure FS</name>
+
+    <packaging>jar</packaging>
+
+    <properties>
+        <fs.azure.sdk.version>3.3.4</fs.azure.sdk.version>
+        <fs.azure.api.version>1.16.0</fs.azure.api.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.fluss</groupId>
+            <artifactId>fluss-common</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- The Hadoop file system adapter classes (bundled) -->
+        <dependency>
+            <groupId>org.apache.fluss</groupId>
+            <artifactId>fluss-fs-hadoop</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.fluss</groupId>
+            <artifactId>fluss-fs-hadoop-shaded</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-azure</artifactId>
+            <version>${fs.hadoopshaded.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.microsoft.azure</groupId>
+                    <artifactId>azure</artifactId>
+                </exclusion>
+                <exclusion>
+                    <!-- provided by fluss-fs-hadoop-shaded -->
+                    <groupId>org.apache.hadoop</groupId>
+                    <artifactId>hadoop-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>ch.qos.reload4j</groupId>
+                    <artifactId>reload4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-reload4j</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.microsoft.azure</groupId>
+            <artifactId>azure</artifactId>
+            <version>${fs.azure.api.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- for the behavior test suite -->
+        <dependency>
+            <groupId>org.apache.fluss</groupId>
+            <artifactId>fluss-common</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+            <type>test-jar</type>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.fluss</groupId>
+            <artifactId>fluss-test-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <archive>
+                        <manifestEntries>
+                            <!-- jaxb-api is packaged as an optional dependency that is only accessible on Java 11 -->
+                            <Multi-Release>true</Multi-Release>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-javax-jars</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <artifactItems>
+                        <artifactItem>
+                            <groupId>javax.xml.bind</groupId>
+                            <artifactId>jaxb-api</artifactId>
+                            <version>${jaxb.api.version}</version>
+                            <type>jar</type>
+                            <overWrite>true</overWrite>
+                        </artifactItem>
+                    </artifactItems>
+                    <outputDirectory>${project.build.directory}/temporary</outputDirectory>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>unpack-javax-libraries</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <target>
+                                <echo message="unpacking javax jars"/>
+                                <unzip dest="${project.build.directory}/classes/META-INF/versions/11">
+                                    <fileset dir="${project.build.directory}/temporary">
+                                        <include name="*"/>
+                                    </fileset>
+                                </unzip>
+                            </target>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>shade-fluss</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <artifactSet>
+                                <includes>
+                                    <include>*:*</include>
+                                </includes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <artifact>*</artifact>
+                                    <excludes>
+                                        <exclude>.gitkeep</exclude>
+                                        <exclude>mime.types</exclude>
+                                        <exclude>mozilla/**</exclude>
+                                        <exclude>META-INF/maven/**</exclude>
+                                        <exclude>META-INF/LICENSE.txt</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <artifact>org.apache.fluss:fluss-fs-hadoop</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/**</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <artifact>*</artifact>
+                                    <excludes>
+                                        <exclude>properties.dtd</exclude>
+                                        <exclude>PropertyList-1.0.dtd</exclude>
+                                        <exclude>META-INF/services/javax.xml.stream.*</exclude>
+                                        <exclude>META-INF/LICENSE.txt</exclude>
+                                        <!-- Remove files tripping CI forbidden GPL regex in transitive deps -->
+                                        <exclude>com/sun/xml/bind/**/Messages.properties</exclude>
+                                        <exclude>com/sun/jersey/json/impl/impl.properties</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

fluss-filesystems/fluss-fs-azure/src/main/java/org/apache/fluss/fs/azure/AbfsFileSystemPlugin.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.fs.azure;
+
+/**
+ * FileSystem plugin for Azure Blob Storage using the ABFS (Azure Blob File System) driver.
+ * Registered for the {@code abfs://} scheme.
+ *
+ * <p>ABFS is the recommended driver for accessing Azure Data Lake Storage Gen2. Use this scheme for
+ * non-SSL connections to ADLS Gen2 storage accounts.
+ *
+ * <p>URI format: {@code abfs://<container>@<storage-account>.dfs.core.windows.net/<path>}
+ */
+public class AbfsFileSystemPlugin extends AzureFileSystemPlugin {
+
+    @Override
+    public String getScheme() {
+        return "abfs";
+    }
+}

fluss-filesystems/fluss-fs-azure/src/main/java/org/apache/fluss/fs/azure/AbfssFileSystemPlugin.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.fs.azure;
+
+/**
+ * FileSystem plugin for Azure Blob Storage using the ABFS driver with SSL/TLS encryption.
+ * Registered for the {@code abfss://} scheme.
+ *
+ * <p>This is the secure (SSL-enabled) variant of ABFS, recommended for production use with Azure
+ * Data Lake Storage Gen2.
+ *
+ * <p>URI format: {@code abfss://<container>@<storage-account>.dfs.core.windows.net/<path>}
+ */
+public class AbfssFileSystemPlugin extends AzureFileSystemPlugin {
+
+    @Override
+    public String getScheme() {
+        return "abfss";
+    }
+}

fluss-filesystems/fluss-fs-azure/src/main/java/org/apache/fluss/fs/azure/AzureFileSystem.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.fs.azure;
+
+import org.apache.fluss.config.Configuration;
+import org.apache.fluss.fs.FileSystem;
+import org.apache.fluss.fs.azure.token.AzureDelegationTokenProvider;
+import org.apache.fluss.fs.hdfs.HadoopFileSystem;
+import org.apache.fluss.fs.token.ObtainedSecurityToken;
+
+import java.io.IOException;
+
+/**
+ * Implementation of the Fluss {@link FileSystem} interface for Azure Blob Storage. This class
+ * implements the common behavior implemented directly by Fluss and delegates common calls to an
+ * implementation of Hadoop's filesystem abstraction.
+ */
+public class AzureFileSystem extends HadoopFileSystem {
+
+    private final String scheme;
+    private final Configuration conf;
+
+    private volatile AzureDelegationTokenProvider delegationTokenProvider;
+
+    /**
+     * Wraps the given Hadoop File System object as a Fluss File System object. The given Hadoop
+     * file system object is expected to be initialized already.
+     *
+     * @param hadoopFileSystem The Hadoop FileSystem that will be used under the hood.
+     */
+    public AzureFileSystem(
+            String scheme, org.apache.hadoop.fs.FileSystem hadoopFileSystem, Configuration conf) {
+        super(hadoopFileSystem);
+        this.scheme = scheme;
+        this.conf = conf;
+    }
+
+    @Override
+    public ObtainedSecurityToken obtainSecurityToken() throws IOException {
+        if (delegationTokenProvider == null) {
+            synchronized (this) {
+                if (delegationTokenProvider == null) {
+                    delegationTokenProvider = new AzureDelegationTokenProvider(scheme, conf);
+                }
+            }
+        }
+
+        return delegationTokenProvider.obtainSecurityToken();
+    }
+}