KNOX-3328: Add Docker-based integration tests for Knox LDAP service

.github/workflows/build/conf/topologies/knoxldap.xml

@@ -35,7 +35,7 @@ limitations under the License.
 			</param>
 			<param>
 				<name>main.ldapRealm.contextFactory.url</name>
-				<value>ldap://ldap:33389</value>
+				<value>ldap://localhost:33390</value>  <!-- Local Knox LDAP service which is configured to proxy to the demo LDAP as a backend -->
 			</param>
 			<param>
 				<name>main.ldapRealm.contextFactory.authenticationMechanism</name>

.github/workflows/build/gateway-site.xml

@@ -160,4 +160,52 @@ limitations under the License.
         <value>max-age=300; includeSubDomains</value>
     </property>
 
+    <!-- KnoxLDAP Service Configuration -->
+    <property>
+        <name>gateway.ldap.enabled</name>
+        <value>true</value>
+    </property>
+    <property>
+        <name>gateway.ldap.port</name>
+        <value>33390</value>
+    </property>
+    <property>
+        <name>gateway.ldap.base.dn</name>
+        <value>dc=hadoop,dc=apache,dc=org</value>
+    </property>
+    <property>
+        <name>gateway.ldap.backend.type</name>
+        <value>ldap</value>
+    </property>
+
+    <!-- LDAP Backend specific configuration (proxying to demo ldap) -->
+    <property>
+        <name>gateway.ldap.backend.ldap.url</name>
+        <value>ldap://ldap:33389</value>
+    </property>
+    <property>
+        <name>gateway.ldap.backend.ldap.remoteBaseDn</name>
+        <value>dc=hadoop,dc=apache,dc=org</value>
+    </property>
+    <property>
+        <name>gateway.ldap.backend.ldap.systemUsername</name>
+        <value>uid=guest,ou=people,dc=hadoop,dc=apache,dc=org</value>
+    </property>
+    <property>
+        <name>gateway.ldap.backend.ldap.systemPassword</name>
+        <value>guest-password</value>
+    </property>
+    <property>
+        <name>gateway.ldap.backend.ldap.userSearchBase</name>
+        <value>ou=people,dc=hadoop,dc=apache,dc=org</value>
+    </property>
+    <property>
+        <name>gateway.ldap.backend.ldap.groupSearchBase</name>
+        <value>ou=groups,dc=hadoop,dc=apache,dc=org</value>
+    </property>
+    <property>
+        <name>gateway.ldap.backend.ldap.groupMemberAttribute</name>
+        <value>member</value>
+    </property>
+
 </configuration>

.github/workflows/tests/test_knox_auth_service_and_LDAP.py

@@ -31,7 +31,7 @@ class TestKnoxAuthService(unittest.TestCase):
     def setUp(self):
         self.base_url = gateway_base_url()
         # The topology name is based on the filename knoxldap.xml
-        self.topology_url = self.base_url + "gateway/knoxldap/auth/api/v1/extauthz"
+        self.topology_url = self.base_url + "gateway/knoxldap/auth/api/v1/pre"
 
     def test_auth_service_guest(self):
         """
@@ -47,7 +47,7 @@ def test_auth_service_guest(self):
         self.assertEqual(response.status_code, 200)
 
         # Check for Actor ID header
-        # The config in knoxtoken.xml sets 'preauth.auth.header.actor.id.name' to 'x-knox-actor-username'
+        # The config in knoxldap.xml sets 'preauth.auth.header.actor.id.name' to 'x-knox-actor-username'
         actor_id_header = 'x-knox-actor-username'
         self.assertIn(actor_id_header, response.headers)
         self.assertEqual(response.headers[actor_id_header], 'guest')

gateway-server/pom.xml

@@ -519,6 +519,12 @@
             <artifactId>api-ldap-client-api</artifactId>
             <scope>provided</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.mina</groupId>
+            <artifactId>mina-core</artifactId>
+        </dependency>
+
         <!-- ********** ********** ********** ********** ********** ********** -->
         <!-- ********** Test Dependencies                           ********** -->
         <!-- ********** ********** ********** ********** ********** ********** -->
@@ -612,10 +618,5 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.mina</groupId>
-            <artifactId>mina-core</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/GroupLookupInterceptor.java

@@ -17,15 +17,19 @@
  */
 package org.apache.knox.gateway.services.ldap;
 
+import org.apache.directory.api.ldap.model.constants.AuthenticationLevel;
 import org.apache.directory.api.ldap.model.cursor.ListCursor;
 import org.apache.directory.api.ldap.model.entry.Entry;
 import org.apache.directory.api.ldap.model.exception.LdapException;
 import org.apache.directory.api.ldap.model.schema.SchemaManager;
+import org.apache.directory.server.core.api.CoreSession;
 import org.apache.directory.server.core.api.DirectoryService;
+import org.apache.directory.server.core.api.LdapPrincipal;
 import org.apache.directory.server.core.api.filtering.EntryFilteringCursor;
 import org.apache.directory.server.core.api.filtering.EntryFilteringCursorImpl;
 import org.apache.directory.server.core.api.interceptor.BaseInterceptor;
 import org.apache.directory.server.core.api.interceptor.context.BindOperationContext;
+import org.apache.directory.server.core.api.interceptor.context.LookupOperationContext;
 import org.apache.directory.server.core.api.interceptor.context.SearchOperationContext;
 import org.apache.knox.gateway.i18n.messages.MessagesFactory;
 import org.apache.knox.gateway.services.ldap.backend.LdapBackend;
@@ -51,6 +55,22 @@ public GroupLookupInterceptor(DirectoryService directoryService, LdapBackend bac
         this.backend = backend;
     }
 
+    @Override
+    public Entry lookup(LookupOperationContext ctx) throws LdapException {
+        Entry entry = next(ctx);
+        if (entry == null) {
+            String username = LdapUtils.extractUsernameFromDn(ctx.getDn());
+            if (username != null) {
+                try {
+                    entry = backend.getUser(username, directoryService.getSchemaManager());
+                } catch (Exception e) {
+                    LOG.ldapServiceStopFailed(e);
+                }
+            }
+        }
+        return entry;
+    }
+
     @Override
     public EntryFilteringCursor search(SearchOperationContext ctx) throws LdapException {
         String filter = ctx.getFilter() != null ? ctx.getFilter().toString() : "";
@@ -120,9 +140,25 @@ public EntryFilteringCursor search(SearchOperationContext ctx) throws LdapExcept
     }
 
     @Override
-    public void bind(BindOperationContext ctx) {
-        // Allow anonymous bind or simple bind
+    public void bind(BindOperationContext ctx) throws LdapException {
         LOG.ldapBind(ctx.getDn() != null ? ctx.getDn().toString() : "anonymous");
+
+        // Try backend first for non-system users
+        if (ctx.getDn() != null && !ctx.getDn().toString().endsWith("ou=system")) {
+            byte[] credentials = ctx.getCredentials();
+            if (credentials != null) {
+                String password = new String(credentials, java.nio.charset.StandardCharsets.UTF_8);
+                if (backend.authenticate(ctx.getDn(), password)) {
+                    // Create session for the authenticated user and set it in context
+                    // This is required by LdapServer to avoid NullPointerException
+                    LdapPrincipal principal = new LdapPrincipal(directoryService.getSchemaManager(), ctx.getDn(), AuthenticationLevel.SIMPLE);
+                    CoreSession session = directoryService.getSession(principal);
+                    ctx.setSession(session);
+                    return; // Successfully authenticated via backend, bypass local check
+                }
+            }
+        }
+
         try {
             next(ctx);
         } catch (Exception e) {

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java

@@ -24,6 +24,7 @@
 import org.apache.directory.server.core.DefaultDirectoryService;
 import org.apache.directory.server.core.api.DirectoryService;
 import org.apache.directory.server.core.api.InstanceLayout;
+import org.apache.directory.server.core.api.interceptor.Interceptor;
 import org.apache.directory.server.core.api.schema.SchemaPartition;
 import org.apache.directory.server.core.partition.impl.btree.jdbm.JdbmPartition;
 import org.apache.directory.server.core.partition.ldif.LdifPartition;
@@ -34,6 +35,8 @@
 import org.apache.knox.gateway.services.ldap.backend.LdapBackend;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -125,8 +128,7 @@ public void start() throws Exception {
             directoryService.addPartition(remotePartition);
         }
 
-        // Add our interceptor for group lookups
-        directoryService.addLast(new GroupLookupInterceptor(directoryService, backend));
+        addGroupLookupInterceptor();
 
         // Allow anonymous access
         directoryService.setAllowAnonymousAccess(true);
@@ -147,6 +149,27 @@ public void start() throws Exception {
         LOG.ldapServiceStarted(port);
     }
 
+    private void addGroupLookupInterceptor() {
+        // Add our interceptor for group lookups and bind proxying
+        // We need to insert it before AuthenticationInterceptor to intercept bind requests
+        final List<Interceptor> interceptors = new ArrayList<>(directoryService.getInterceptors());
+        int authIdx = -1;
+        for (int i = 0; i < interceptors.size(); i++) {
+            if (interceptors.get(i).getName().equalsIgnoreCase("authenticationInterceptor")) {
+                authIdx = i;
+                break;
+            }
+        }
+
+        final GroupLookupInterceptor interceptor = new GroupLookupInterceptor(directoryService, backend);
+        if (authIdx != -1) {
+            interceptors.add(authIdx, interceptor);
+        } else {
+            interceptors.add(interceptor);
+        }
+        directoryService.setInterceptors(interceptors);
+    }
+
     /**
      * Stop the LDAP server
      */

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java

@@ -96,4 +96,10 @@ public interface LdapMessages {
     @Message(level = MessageLevel.ERROR,
             text = "Failed to copy attribute: {0}")
     void ldapAttributeCopyError(@StackTrace(level = MessageLevel.DEBUG) Exception e);
+
+    @Message(level = MessageLevel.DEBUG, text = "LDAP authentication succeeded for user: {0}")
+    void ldapAuthSucceeded(String user);
+
+    @Message(level = MessageLevel.WARN, text = "LDAP authentication failed for user: {0}")
+    void ldapAuthFailed(String user, @StackTrace(level = MessageLevel.INFO) Throwable cause);
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapUtils.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.services.ldap;
+
+import org.apache.directory.api.ldap.model.name.Dn;
+
+public class LdapUtils {
+
+    public static String extractUsernameFromDn(Dn dn) {
+        if (dn == null || dn.isEmpty()) {
+            return null;
+        }
+
+        try {
+            return "uid".equalsIgnoreCase(dn.getRdn().getType())
+                    ? dn.getRdn().getValue()
+                    : null;
+        } catch (Exception ignored) {
+            return null;
+        }
+    }
+}

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/FileBackend.java

@@ -20,9 +20,11 @@
 import com.google.gson.Gson;
 import org.apache.directory.api.ldap.model.entry.DefaultEntry;
 import org.apache.directory.api.ldap.model.entry.Entry;
+import org.apache.directory.api.ldap.model.name.Dn;
 import org.apache.directory.api.ldap.model.schema.SchemaManager;
 import org.apache.knox.gateway.i18n.messages.MessagesFactory;
 import org.apache.knox.gateway.services.ldap.LdapMessages;
+import org.apache.knox.gateway.services.ldap.LdapUtils;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -45,6 +47,7 @@ public class FileBackend implements LdapBackend {
 
     static class UserData {
         String username;
+        String password;
         String cn;
         String sn;
         List<String> groups;
@@ -141,4 +144,17 @@ public List<Entry> searchUsers(String filter, SchemaManager schemaManager) throw
 
         return results;
     }
+
+    @Override
+    public boolean authenticate(Dn userDn, String password) {
+        // Extract username from DN (e.g., uid=admin,  ou=people,dc=hadoop,dc=apache,dc=org)
+        final String username = LdapUtils.extractUsernameFromDn(userDn);
+
+        if (username != null) {
+            UserData userData = users.get(username);
+            return userData != null && password != null && password.equals(userData.password);
+        }
+
+        return false;
+    }
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapBackend.java

@@ -18,6 +18,7 @@
 package org.apache.knox.gateway.services.ldap.backend;
 
 import org.apache.directory.api.ldap.model.entry.Entry;
+import org.apache.directory.api.ldap.model.name.Dn;
 import org.apache.directory.api.ldap.model.schema.SchemaManager;
 
 import java.util.List;
@@ -65,4 +66,13 @@ public interface LdapBackend {
      * @return List of matching entries
      */
     List<Entry> searchUsers(String filter, SchemaManager schemaManager) throws Exception;
+
+    /**
+     * Authenticate a user with password
+     *
+     * @param userDn   The user's Distinguished Name
+     * @param password The user's password
+     * @return true if authentication is successful, false otherwise
+     */
+    boolean authenticate(Dn userDn, String password);
 }

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java

@@ -24,11 +24,13 @@
 import org.apache.directory.api.ldap.model.entry.Entry;
 import org.apache.directory.api.ldap.model.exception.LdapException;
 import org.apache.directory.api.ldap.model.message.SearchScope;
+import org.apache.directory.api.ldap.model.name.Dn;
 import org.apache.directory.api.ldap.model.schema.SchemaManager;
 import org.apache.directory.ldap.client.api.DefaultLdapConnectionFactory;
 import org.apache.directory.ldap.client.api.LdapConnection;
 import org.apache.directory.ldap.client.api.LdapConnectionConfig;
 import org.apache.directory.ldap.client.api.LdapConnectionPool;
+import org.apache.directory.ldap.client.api.LdapNetworkConnection;
 import org.apache.directory.ldap.client.api.ValidatingPoolableLdapConnectionFactory;
 import org.apache.knox.gateway.i18n.messages.MessagesFactory;
 import org.apache.knox.gateway.services.ldap.LdapMessages;
@@ -253,6 +255,25 @@ public void close() {
         }
     }
 
+    @Override
+    public boolean authenticate(Dn userDn, String password) {
+        final String userDnText = userDn.toString(); //at this point we are sure it's not NULL
+        // Create a temporary connection for authentication (bind)
+        final LdapConnectionConfig authConfig = new LdapConnectionConfig();
+        authConfig.setLdapHost(host);
+        authConfig.setLdapPort(port);
+        authConfig.setName(userDnText);
+        authConfig.setCredentials(password);
+        try (LdapConnection connection =  new LdapNetworkConnection(authConfig)){
+            connection.bind();
+            LOG.ldapAuthSucceeded(userDnText);
+            return true;
+        } catch (Exception e) {
+            LOG.ldapAuthFailed(userDnText, e);
+            return false;
+        }
+    }
+
     @Override
     public Entry getUser(String username, SchemaManager schemaManager) throws Exception {
         LdapConnection connection = null;

pom.xml

@@ -800,6 +800,7 @@
                                 <ignoredNonTestScopedDependency>com.nimbusds:nimbus-jose-jwt</ignoredNonTestScopedDependency>
                                 <ignoredNonTestScopedDependency>org.apache.httpcomponents:httpcore</ignoredNonTestScopedDependency>
                                 <ignoredNonTestScopedDependency>org.apache.knox:gateway-server</ignoredNonTestScopedDependency>
+                                <ignoredNonTestScopedDependency>org.apache.mina:mina-core</ignoredNonTestScopedDependency>
                             </ignoredNonTestScopedDependencies>
                             <ignoredUsedUndeclaredDependencies>
                                 <ignoredUsedUndeclaredDependency>org.glassfish.hk2.external:jakarta.inject</ignoredUsedUndeclaredDependency>