diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5280e4c735b..834b7933cda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: strategy: matrix: # Java versions to run unit tests (Jetty 12 requires Java 17+) - java: [ '17', '21' ] + java: [ '17', '21', '25' ] profile: ['default-hadoop'] fail-fast: false steps: diff --git a/contrib/storage-phoenix/pom.xml b/contrib/storage-phoenix/pom.xml index 3ac604f627b..f12766ee752 100644 --- a/contrib/storage-phoenix/pom.xml +++ b/contrib/storage-phoenix/pom.xml @@ -335,6 +335,9 @@ -Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.krb5.allowUdp=false + + -Dorg.apache.hbase.thirdparty.io.netty.noUnsafe=false diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AsmUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AsmUtil.java index fd071d62307..eff29e088ed 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AsmUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/AsmUtil.java @@ -53,7 +53,7 @@ public static boolean isClassOk(final Logger logger, final String logTag, final classNode.accept(verifyWriter); final ClassReader ver = new ClassReader(verifyWriter.toByteArray()); try { - DrillCheckClassAdapter.verify(ver, false, new PrintWriter(sw)); + DrillCheckClassAdapter.verify(ver, new PrintWriter(sw)); } catch(final Exception e) { logger.info("Caught exception verifying class:"); logClass(logger, logTag, classNode); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillCheckClassAdapter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillCheckClassAdapter.java index 4076c2379cc..bf632d70041 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillCheckClassAdapter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/DrillCheckClassAdapter.java @@ -18,10 +18,20 @@ package org.apache.drill.exec.compile; import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.tree.ClassNode; +import org.objectweb.asm.tree.MethodNode; +import org.objectweb.asm.tree.analysis.Analyzer; +import org.objectweb.asm.tree.analysis.AnalyzerException; +import org.objectweb.asm.tree.analysis.BasicValue; +import org.objectweb.asm.tree.analysis.SimpleVerifier; import org.objectweb.asm.util.CheckClassAdapter; /** @@ -100,19 +110,79 @@ protected DrillCheckClassAdapter(final int api, final ClassVisitor cv, } /** - * See {@link org.objectweb.asm.util.CheckClassAdapter#verify(ClassReader, boolean, PrintWriter)}. + * Data flow verification, equivalent to + * {@link org.objectweb.asm.util.CheckClassAdapter#verify(ClassReader, boolean, PrintWriter)} + * but tolerant of types that cannot be loaded (see {@link LenientVerifier}). + * Any problem found is written to pw; nothing is written if the + * class is well formed. */ - public static void verify(final ClassReader cr, final boolean dump, - final PrintWriter pw) { + public static void verify(final ClassReader cr, final PrintWriter pw) { /* * For plain verification, we don't need to restore the original access * bytes the way we do when the check adapter is used as part of a chain, so - * we can just strip it and use the ASM version directly. + * we can just strip it and verify directly. */ final ClassWriter classWriter = new ClassWriter(0); cr.accept(new InnerClassAccessStripper(CompilationConfig.ASM_API_VERSION, classWriter), ClassReader.SKIP_DEBUG); - final ClassReader strippedCr = new ClassReader(classWriter.toByteArray()); - CheckClassAdapter.verify(strippedCr, dump, pw); + + final ClassNode classNode = new ClassNode(); + new ClassReader(classWriter.toByteArray()).accept(classNode, ClassReader.SKIP_DEBUG); + + final Type currentClass = Type.getObjectType(classNode.name); + final Type currentSuperClass = + classNode.superName == null ? null : Type.getObjectType(classNode.superName); + final List currentClassInterfaces = new ArrayList<>(); + for (String interfaceName : classNode.interfaces) { + currentClassInterfaces.add(Type.getObjectType(interfaceName)); + } + final boolean isInterface = (classNode.access & Opcodes.ACC_INTERFACE) != 0; + + for (MethodNode method : classNode.methods) { + final SimpleVerifier verifier = new LenientVerifier( + currentClass, currentSuperClass, currentClassInterfaces, isInterface); + try { + new Analyzer<>(verifier).analyze(classNode.name, method); + } catch (AnalyzerException e) { + e.printStackTrace(pw); + } + } + } + + /** + * ASM's {@link SimpleVerifier} resolves types with {@link Class#forName}, which + * cannot work for the classes Drill is in the middle of generating: a generated + * nested class refers to its enclosing generated class, and neither has been + * defined in any class loader yet. Since JDK 22 javac emits an + * Objects.requireNonNull(outer) prologue in nested class + * constructors, which makes the verifier resolve the enclosing class and fail. + * + *

Types that cannot be loaded are treated as assignable, so verification + * still covers everything that is resolvable. + */ + private static class LenientVerifier extends SimpleVerifier { + LenientVerifier(final Type currentClass, final Type currentSuperClass, + final List currentClassInterfaces, final boolean isInterface) { + super(CompilationConfig.ASM_API_VERSION, currentClass, currentSuperClass, + currentClassInterfaces, isInterface); + } + + @Override + protected boolean isAssignableFrom(final Type type1, final Type type2) { + try { + return super.isAssignableFrom(type1, type2); + } catch (TypeNotPresentException e) { + return true; + } + } + + @Override + public BasicValue merge(final BasicValue value1, final BasicValue value2) { + try { + return super.merge(value1, value2); + } catch (TypeNotPresentException e) { + return BasicValue.REFERENCE_VALUE; + } + } } } diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java index 98b4793d75d..777a6379779 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java @@ -26,9 +26,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.HadoopKerberosName; +import org.apache.hadoop.security.authentication.util.SubjectUtil; import org.apache.hadoop.security.UserGroupInformation; -import javax.security.auth.Subject; import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; import javax.security.auth.callback.UnsupportedCallbackException; @@ -41,7 +41,6 @@ import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.UndeclaredThrowableException; -import java.security.AccessController; import java.security.PrivilegedExceptionAction; import java.util.Map; @@ -68,7 +67,7 @@ public UserGroupInformation createAndLoginUser(final Map properties) try { final UserGroupInformation ugi; if (assumeSubject) { - ugi = UserGroupInformation.getUGIFromSubject(Subject.getSubject(AccessController.getContext())); + ugi = UserGroupInformation.getUGIFromSubject(SubjectUtil.current()); logger.debug("Assuming subject for {}.", ugi.getShortUserName()); } else { if (keytab != null) { diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java index 3f12a1e1d6d..c2be2e15255 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/user/UserClient.java @@ -79,10 +79,14 @@ import org.apache.drill.exec.ssl.SSLConfig; import org.apache.drill.exec.ssl.SSLConfigBuilder; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.SubjectUtil; import org.slf4j.Logger; import javax.net.ssl.SSLEngine; +import javax.security.auth.Subject; import javax.security.sasl.SaslException; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; import java.io.IOException; import java.util.List; import java.util.Map; @@ -110,6 +114,11 @@ public class UserClient private DrillProperties properties; + // ponytail: the SASL handshake completes on a Netty thread, which no longer inherits the caller's + // Subject (JEP 486 replaced the inheritable AccessControlContext with a scoped value). Capture the + // Subject on the connecting thread and rebind it around the login below. + private Subject subject; + public UserClient(String clientName, DrillConfig config, Properties properties, boolean supportComplexTypes, BufferAllocator allocator, EventLoopGroup eventLoopGroup, Executor eventExecutor, DrillbitEndpoint endpoint) throws NonTransientRpcException { @@ -174,6 +183,7 @@ public void submitQuery(UserResultsListener resultsListener, RunQuery query) { */ public void connect(final DrillbitEndpoint endpoint, final DrillProperties properties, final UserCredentials credentials) throws RpcException { + subject = SubjectUtil.current(); final UserToBitHandshake.Builder hsBuilder = UserToBitHandshake.newBuilder() .setRpcVersion(UserRpcConfig.RPC_VERSION) @@ -449,7 +459,15 @@ protected void prepareSaslHandshake(final RpcConnectionHandler) () -> factory.createAndLoginUser(saslProperties)); + } catch (PrivilegedActionException e) { + Thread.currentThread().setContextClassLoader(oldThreadCtxtCL); + throw e.getCause() instanceof IOException + ? (IOException) e.getCause() : new IOException(e.getCause()); + } // Reset the thread context class loader to original one Thread.currentThread().setContextClassLoader(oldThreadCtxtCL); diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java index 879dc4d1561..500a4b1d021 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java @@ -73,7 +73,7 @@ private static final void check(final byte[] b) { final StringWriter sw = new StringWriter(); final PrintWriter pw = new PrintWriter(sw); - DrillCheckClassAdapter.verify(new ClassReader(cw.toByteArray()), false, pw); + DrillCheckClassAdapter.verify(new ClassReader(cw.toByteArray()), pw); final String checkString = sw.toString(); if (!checkString.isEmpty()) { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java index 0783b44982b..39183d87e0f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberos.java @@ -40,6 +40,7 @@ import org.junit.experimental.categories.Category; import javax.security.auth.Subject; +import org.apache.hadoop.security.authentication.util.SubjectUtil; import java.security.PrivilegedExceptionAction; import static junit.framework.TestCase.assertEquals; @@ -100,7 +101,7 @@ public void successTicket() throws Exception { ); try ( - ClientFixture client = Subject.doAs( + ClientFixture client = SubjectUtil.doAs( clientSubject, (PrivilegedExceptionAction) () -> cluster.clientBuilder() .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL) @@ -136,7 +137,7 @@ public void testUnencryptedConnectionCounter() throws Exception { try ( // Use a dedicated cluster fixture so that the tested RPC counters have a clean start. ClusterFixture cluster = defaultClusterConfig().build(); - ClientFixture client = Subject.doAs( + ClientFixture client = SubjectUtil.doAs( clientSubject, (PrivilegedExceptionAction) () -> cluster.clientBuilder() .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL) @@ -178,7 +179,7 @@ public void testUnencryptedConnectionCounter_LocalControlMessage() throws Except try ( // Use a dedicated cluster fixture so that the tested RPC counters have a clean start. ClusterFixture cluster = defaultClusterConfig().build(); - ClientFixture client = Subject.doAs( + ClientFixture client = SubjectUtil.doAs( clientSubject, (PrivilegedExceptionAction) () -> cluster.clientBuilder() .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java index 5f7b0f39caa..560dd46e44d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/rpc/user/security/TestUserBitKerberosEncryption.java @@ -42,6 +42,7 @@ import org.junit.experimental.categories.Category; import javax.security.auth.Subject; +import org.apache.hadoop.security.authentication.util.SubjectUtil; import java.security.PrivilegedExceptionAction; import static junit.framework.TestCase.assertEquals; @@ -157,7 +158,7 @@ public void successTicketWithoutChunking() throws Exception { ); try ( - ClientFixture client = Subject.doAs( + ClientFixture client = SubjectUtil.doAs( clientSubject, (PrivilegedExceptionAction) () -> cluster.clientBuilder() .property(DrillProperties.SERVICE_PRINCIPAL, krbHelper.SERVER_PRINCIPAL) diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java index c0b8b617c00..5d86c0f08f9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestDrillSpnegoAuthenticator.java @@ -45,6 +45,7 @@ import org.junit.experimental.categories.Category; import javax.security.auth.Subject; +import org.apache.hadoop.security.authentication.util.SubjectUtil; import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.concurrent.TimeUnit; @@ -118,7 +119,7 @@ private String generateSpnegoToken() throws Exception { final Subject clientSubject = JaasKrbUtil.loginUsingKeytab(spnegoHelper.CLIENT_PRINCIPAL, spnegoHelper.clientKeytab.getAbsoluteFile()); - return Subject.doAs(clientSubject, (PrivilegedExceptionAction) () -> { + return SubjectUtil.doAs(clientSubject, (PrivilegedExceptionAction) () -> { final GSSManager gssManager = GSSManager.getInstance(); GSSContext gssContext = null; try { diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java index cf8f38b84a6..678f35574c6 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/server/rest/spnego/TestSpnegoAuthentication.java @@ -53,6 +53,7 @@ import org.mockito.Mockito; import javax.security.auth.Subject; +import org.apache.hadoop.security.authentication.util.SubjectUtil; import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; @@ -255,7 +256,7 @@ public void testDrillSpnegoLoginService() throws Exception { spnegoHelper.clientKeytab.getAbsoluteFile()); // Generate a SPNEGO token for the peer SERVER_PRINCIPAL from this CLIENT_PRINCIPAL - final String token = Subject.doAs(clientSubject, new PrivilegedExceptionAction() { + final String token = SubjectUtil.doAs(clientSubject, new PrivilegedExceptionAction() { @Override public String run() throws Exception { diff --git a/pom.xml b/pom.xml index 1a107272670..f69207ab62c 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,9 @@ 1 2.3.30 32.1.2-jre - 3.4.1 + + 3.4.3 2.2 2.6.1-hadoop3 4.0.3 @@ -123,8 +125,7 @@ 3.8.4 4096 4.2.19 - 5.17.0 - 5.2.0 + 5.23.0 5.5.1 0.6.6 15.4 @@ -510,7 +511,7 @@ [${maven.version.min},4) - [17,24) + [17,26) @@ -932,18 +933,6 @@ test - - org.mockito - mockito-inline - ${mockito_inline.version} - test - - - mockito-core - org.mockito - - - de.huxhorn.lilith de.huxhorn.lilith.logback.appender.multiplex-classic