DRILL-8553: Add Support for Java 25 LTS

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         # Java versions to run unit tests (Jetty 12 requires Java 17+)
-        java: [ '17', '21' ]
+        java: [ '17', '21', '25' ]
         profile: ['default-hadoop']
       fail-fast: false
     steps:

contrib/storage-phoenix/pom.xml

@@ -335,6 +335,9 @@
             -Djava.net.preferIPv4Stack=true
             -Dsun.security.krb5.debug=true
             -Dsun.security.krb5.allowUdp=false
+            <!-- HBase's shaded Netty disables sun.misc.Unsafe by default on Java 24+, but HBase's own
+                 ByteBuff/UnsafeAccess still calls into it, so the mini cluster NPEs on every RPC. -->
+            -Dorg.apache.hbase.thirdparty.io.netty.noUnsafe=false
           </argLine>
         </configuration>
       </plugin>

exec/java-exec/src/main/java/org/apache/drill/exec/compile/AsmUtil.java

@@ -53,7 +53,7 @@ public static boolean isClassOk(final Logger logger, final String logTag, final
     classNode.accept(verifyWriter);
     final ClassReader ver = new ClassReader(verifyWriter.toByteArray());
     try {
-      DrillCheckClassAdapter.verify(ver, false, new PrintWriter(sw));
+      DrillCheckClassAdapter.verify(ver, new PrintWriter(sw));
     } catch(final Exception e) {
       logger.info("Caught exception verifying class:");
       logClass(logger, logTag, classNode);

exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillCheckClassAdapter.java

@@ -18,10 +18,20 @@
 package org.apache.drill.exec.compile;
 
 import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.tree.ClassNode;
+import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.tree.analysis.Analyzer;
+import org.objectweb.asm.tree.analysis.AnalyzerException;
+import org.objectweb.asm.tree.analysis.BasicValue;
+import org.objectweb.asm.tree.analysis.SimpleVerifier;
 import org.objectweb.asm.util.CheckClassAdapter;
 
 /**
@@ -100,19 +110,79 @@ protected DrillCheckClassAdapter(final int api, final ClassVisitor cv,
   }
 
   /**
-   * See {@link org.objectweb.asm.util.CheckClassAdapter#verify(ClassReader, boolean, PrintWriter)}.
+   * Data flow verification, equivalent to
+   * {@link org.objectweb.asm.util.CheckClassAdapter#verify(ClassReader, boolean, PrintWriter)}
+   * but tolerant of types that cannot be loaded (see {@link LenientVerifier}).
+   * Any problem found is written to <code>pw</code>; nothing is written if the
+   * class is well formed.
    */
-  public static void verify(final ClassReader cr, final boolean dump,
-      final PrintWriter pw) {
+  public static void verify(final ClassReader cr, final PrintWriter pw) {
     /*
      * For plain verification, we don't need to restore the original access
      * bytes the way we do when the check adapter is used as part of a chain, so
-     * we can just strip it and use the ASM version directly.
+     * we can just strip it and verify directly.
      */
     final ClassWriter classWriter = new ClassWriter(0);
     cr.accept(new InnerClassAccessStripper(CompilationConfig.ASM_API_VERSION,
         classWriter), ClassReader.SKIP_DEBUG);
-    final ClassReader strippedCr = new ClassReader(classWriter.toByteArray());
-    CheckClassAdapter.verify(strippedCr, dump, pw);
+
+    final ClassNode classNode = new ClassNode();
+    new ClassReader(classWriter.toByteArray()).accept(classNode, ClassReader.SKIP_DEBUG);
+
+    final Type currentClass = Type.getObjectType(classNode.name);
+    final Type currentSuperClass =
+        classNode.superName == null ? null : Type.getObjectType(classNode.superName);
+    final List<Type> currentClassInterfaces = new ArrayList<>();
+    for (String interfaceName : classNode.interfaces) {
+      currentClassInterfaces.add(Type.getObjectType(interfaceName));
+    }
+    final boolean isInterface = (classNode.access & Opcodes.ACC_INTERFACE) != 0;
+
+    for (MethodNode method : classNode.methods) {
+      final SimpleVerifier verifier = new LenientVerifier(
+          currentClass, currentSuperClass, currentClassInterfaces, isInterface);
+      try {
+        new Analyzer<>(verifier).analyze(classNode.name, method);
+      } catch (AnalyzerException e) {
+        e.printStackTrace(pw);
+      }
+    }
+  }
+
+  /**
+   * ASM's {@link SimpleVerifier} resolves types with {@link Class#forName}, which
+   * cannot work for the classes Drill is in the middle of generating: a generated
+   * nested class refers to its enclosing generated class, and neither has been
+   * defined in any class loader yet. Since JDK 22 javac emits an
+   * <code>Objects.requireNonNull(outer)</code> prologue in nested class
+   * constructors, which makes the verifier resolve the enclosing class and fail.
+   *
+   * <p>Types that cannot be loaded are treated as assignable, so verification
+   * still covers everything that is resolvable.
+   */
+  private static class LenientVerifier extends SimpleVerifier {
+    LenientVerifier(final Type currentClass, final Type currentSuperClass,
+        final List<Type> currentClassInterfaces, final boolean isInterface) {
+      super(CompilationConfig.ASM_API_VERSION, currentClass, currentSuperClass,
+          currentClassInterfaces, isInterface);
+    }
+
+    @Override
+    protected boolean isAssignableFrom(final Type type1, final Type type2) {
+      try {
+        return super.isAssignableFrom(type1, type2);
+      } catch (TypeNotPresentException e) {
+        return true;
+      }
+    }
+
+    @Override
+    public BasicValue merge(final BasicValue value1, final BasicValue value2) {
+      try {
+        return super.merge(value1, value2);
+      } catch (TypeNotPresentException e) {
+        return BasicValue.REFERENCE_VALUE;
+      }
+    }
   }
 }

exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java

@@ -26,9 +26,9 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.authentication.util.SubjectUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 
-import javax.security.auth.Subject;
 import javax.security.auth.callback.Callback;
 import javax.security.auth.callback.CallbackHandler;
 import javax.security.auth.callback.UnsupportedCallbackException;
@@ -41,7 +41,6 @@
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.UndeclaredThrowableException;
-import java.security.AccessController;
 import java.security.PrivilegedExceptionAction;
 import java.util.Map;
 
@@ -68,7 +67,7 @@ public UserGroupInformation createAndLoginUser(final Map<String, ?> properties)
     try {
       final UserGroupInformation ugi;
       if (assumeSubject) {
-        ugi = UserGroupInformation.getUGIFromSubject(Subject.getSubject(AccessController.getContext()));
+        ugi = UserGroupInformation.getUGIFromSubject(SubjectUtil.current());
         logger.debug("Assuming subject for {}.", ugi.getShortUserName());
       } else {
         if (keytab != null) {

exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java

@@ -79,10 +79,14 @@
 import org.apache.drill.exec.ssl.SSLConfig;
 import org.apache.drill.exec.ssl.SSLConfigBuilder;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.SubjectUtil;
 import org.slf4j.Logger;
 
 import javax.net.ssl.SSLEngine;
+import javax.security.auth.Subject;
 import javax.security.sasl.SaslException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
@@ -110,6 +114,11 @@ public class UserClient
 
   private DrillProperties properties;
 
+  // ponytail: the SASL handshake completes on a Netty thread, which no longer inherits the caller's
+  // Subject (JEP 486 replaced the inheritable AccessControlContext with a scoped value). Capture the
+  // Subject on the connecting thread and rebind it around the login below.
+  private Subject subject;
+
   public UserClient(String clientName, DrillConfig config, Properties properties, boolean supportComplexTypes,
       BufferAllocator allocator, EventLoopGroup eventLoopGroup, Executor eventExecutor,
       DrillbitEndpoint endpoint) throws NonTransientRpcException {
@@ -174,6 +183,7 @@ public void submitQuery(UserResultsListener resultsListener, RunQuery query) {
    */
   public void connect(final DrillbitEndpoint endpoint, final DrillProperties properties,
       final UserCredentials credentials) throws RpcException {
+    subject = SubjectUtil.current();
     final UserToBitHandshake.Builder hsBuilder =
         UserToBitHandshake.newBuilder()
             .setRpcVersion(UserRpcConfig.RPC_VERSION)
@@ -449,7 +459,15 @@ protected void prepareSaslHandshake(final RpcConnectionHandler<UserToBitConnecti
       final ClassLoader oldThreadCtxtCL = Thread.currentThread().getContextClassLoader();
       final ClassLoader newThreadCtxtCL = this.getClass().getClassLoader();
       Thread.currentThread().setContextClassLoader(newThreadCtxtCL);
-      final UserGroupInformation ugi = factory.createAndLoginUser(saslProperties);
+      final UserGroupInformation ugi;
+      try {
+        ugi = SubjectUtil.doAs(subject,
+          (PrivilegedExceptionAction<UserGroupInformation>) () -> factory.createAndLoginUser(saslProperties));
+      } catch (PrivilegedActionException e) {
+        Thread.currentThread().setContextClassLoader(oldThreadCtxtCL);
+        throw e.getCause() instanceof IOException
+          ? (IOException) e.getCause() : new IOException(e.getCause());
+      }
       // Reset the thread context class loader to original one
       Thread.currentThread().setContextClassLoader(oldThreadCtxtCL);
 

exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java

@@ -73,7 +73,7 @@ private static final void check(final byte[] b) {
 
     final StringWriter sw = new StringWriter();
     final PrintWriter pw = new PrintWriter(sw);
-    DrillCheckClassAdapter.verify(new ClassReader(cw.toByteArray()), false, pw);
+    DrillCheckClassAdapter.verify(new ClassReader(cw.toByteArray()), pw);
 
     final String checkString = sw.toString();
     if (!checkString.isEmpty()) {

exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java

@@ -40,6 +40,7 @@
 import org.junit.experimental.categories.Category;
 
 import javax.security.auth.Subject;
+import org.apache.hadoop.security.authentication.util.SubjectUtil;
 import java.security.PrivilegedExceptionAction;
 
 import static junit.framework.TestCase.assertEquals;
@@ -100,7 +101,7 @@ public void successTicket() throws Exception {
     );
 
     try (
-      ClientFixture client = Subject.doAs(
+      ClientFixture client = SubjectUtil.doAs(
         clientSubject,
         (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
           .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
@@ -136,7 +137,7 @@ public void testUnencryptedConnectionCounter() throws Exception {
     try (
       // Use a dedicated cluster fixture so that the tested RPC counters have a clean start.
       ClusterFixture cluster = defaultClusterConfig().build();
-      ClientFixture client = Subject.doAs(
+      ClientFixture client = SubjectUtil.doAs(
         clientSubject,
         (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
           .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)
@@ -178,7 +179,7 @@ public void testUnencryptedConnectionCounter_LocalControlMessage() throws Except
     try (
       // Use a dedicated cluster fixture so that the tested RPC counters have a clean start.
       ClusterFixture cluster = defaultClusterConfig().build();
-      ClientFixture client = Subject.doAs(
+      ClientFixture client = SubjectUtil.doAs(
         clientSubject,
         (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
           .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)

exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java

@@ -42,6 +42,7 @@
 import org.junit.experimental.categories.Category;
 
 import javax.security.auth.Subject;
+import org.apache.hadoop.security.authentication.util.SubjectUtil;
 import java.security.PrivilegedExceptionAction;
 
 import static junit.framework.TestCase.assertEquals;
@@ -157,7 +158,7 @@ public void successTicketWithoutChunking() throws Exception {
     );
 
     try (
-      ClientFixture client = Subject.doAs(
+      ClientFixture client = SubjectUtil.doAs(
         clientSubject,
         (PrivilegedExceptionAction<ClientFixture>) () -> cluster.clientBuilder()
           .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL)

exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java

@@ -45,6 +45,7 @@
 import org.junit.experimental.categories.Category;
 
 import javax.security.auth.Subject;
+import org.apache.hadoop.security.authentication.util.SubjectUtil;
 import java.lang.reflect.Field;
 import java.security.PrivilegedExceptionAction;
 import java.util.concurrent.TimeUnit;
@@ -118,7 +119,7 @@ private String generateSpnegoToken() throws Exception {
     final Subject clientSubject = JaasKrbUtil.loginUsingKeytab(spnegoHelper.CLIENT_PRINCIPAL,
         spnegoHelper.clientKeytab.getAbsoluteFile());
 
-    return Subject.doAs(clientSubject, (PrivilegedExceptionAction<String>) () -> {
+    return SubjectUtil.doAs(clientSubject, (PrivilegedExceptionAction<String>) () -> {
       final GSSManager gssManager = GSSManager.getInstance();
       GSSContext gssContext = null;
       try {

exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java

@@ -53,6 +53,7 @@
 import org.mockito.Mockito;
 
 import javax.security.auth.Subject;
+import org.apache.hadoop.security.authentication.util.SubjectUtil;
 import java.lang.reflect.Field;
 import java.security.PrivilegedExceptionAction;
 
@@ -255,7 +256,7 @@ public void testDrillSpnegoLoginService() throws Exception {
       spnegoHelper.clientKeytab.getAbsoluteFile());
 
     // Generate a SPNEGO token for the peer SERVER_PRINCIPAL from this CLIENT_PRINCIPAL
-    final String token = Subject.doAs(clientSubject, new PrivilegedExceptionAction<String>() {
+    final String token = SubjectUtil.doAs(clientSubject, new PrivilegedExceptionAction<String>() {
       @Override
       public String run() throws Exception {
 

pom.xml

@@ -82,7 +82,9 @@
     <forkCount>1</forkCount>
     <freemarker.version>2.3.30</freemarker.version>
     <guava.version>32.1.2-jre</guava.version>
-    <hadoop.version>3.4.1</hadoop.version>
+    <!-- 3.4.3+ required on JDK 24+: earlier versions call Subject.getSubject(AccessControlContext),
+         which JEP 486 made throw UnsupportedOperationException unconditionally. -->
+    <hadoop.version>3.4.3</hadoop.version>
     <hamcrest.version>2.2</hamcrest.version>
     <hbase.version>2.6.1-hadoop3</hbase.version>
     <hikari.version>4.0.3</hikari.version>
@@ -123,8 +125,7 @@
     <maven.version>3.8.4</maven.version>
     <memoryMb>4096</memoryMb>
     <metrics.version>4.2.19</metrics.version>
-    <mockito.version>5.17.0</mockito.version>
-    <mockito_inline.version>5.2.0</mockito_inline.version>
+    <mockito.version>5.23.0</mockito.version>
     <mongo.version>5.5.1</mongo.version>
     <msgpack.version>0.6.6</msgpack.version>
     <nashorn.version>15.4</nashorn.version>
@@ -510,7 +511,7 @@
                   <version>[${maven.version.min},4)</version>
                 </requireMavenVersion>
                 <requireJavaVersion>
-                  <version>[17,24)</version>
+                  <version>[17,26)</version>
                 </requireJavaVersion>
               </rules>
             </configuration>
@@ -932,18 +933,6 @@
       </exclusions>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-inline</artifactId>
-      <version>${mockito_inline.version}</version>
-      <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>mockito-core</artifactId>
-          <groupId>org.mockito</groupId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <dependency>
       <groupId>de.huxhorn.lilith</groupId>
       <artifactId>de.huxhorn.lilith.logback.appender.multiplex-classic</artifactId>