[cleanup]Remove Trino/PrestoDB Pulsar plugin from main repo (#21795)

Author: merlimat

.github/workflows/ci-owasp-dependency-check.yaml

@@ -84,9 +84,6 @@ jobs:
       - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true)
         run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true
 
-      - name: run OWASP Dependency Check for distribution/offloaders, distribution/io and pulsar-sql/presto-distribution
-        run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io,pulsar-sql/presto-distribution
-
       - name: Upload OWASP Dependency Check reports
         uses: actions/upload-artifact@v3
         if: always()
@@ -96,4 +93,3 @@ jobs:
             distribution/server/target/dependency-check-report.html
             distribution/offloaders/target/dependency-check-report.html
             distribution/io/target/dependency-check-report.html
-            pulsar-sql/presto-distribution/target/dependency-check-report.html

.github/workflows/ci-semantic-pull-request.yml

@@ -53,7 +53,6 @@ jobs:
           # io      -> Pulsar Connectors
           # offload -> tiered storage
           # sec     -> security
-          # sql     -> Pulsar Trino Plugin
           # txn     -> transaction
           # ws      -> websocket
           # ml      -> managed ledger
@@ -79,7 +78,6 @@ jobs:
             schema
             sec
             site
-            sql
             storage
             test
             txn

.github/workflows/pulsar-ci.yaml

@@ -871,8 +871,8 @@ jobs:
       - name: Build latest-version-image docker image
         run: |
           # build docker image
-          # include building of Pulsar SQL, Connectors, Offloaders and server distros
-          mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
+          # include building of Connectors, Offloaders and server distros
+          mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
           -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" -DIMAGE_JDK_MAJOR_VERSION="${IMAGE_JDK_MAJOR_VERSION}" \
           -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
           -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
@@ -954,9 +954,6 @@ jobs:
             group: PULSAR_IO
             clean_disk: true
 
-          - name: Sql
-            group: SQL
-
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -1381,11 +1378,11 @@ jobs:
         run: |
           cd $HOME
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
-      # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
+      # Projects dependent on flume, hdfs, and hbase currently excluded from the scan.
       - name: trigger dependency check
         run: |
           mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
-            -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
+            -pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
 
       - name: Upload report
         uses: actions/upload-artifact@v3

.idea/vcs.xml

@@ -44,9 +44,6 @@ DEFAULT_PY_INSTANCE_FILE=$PULSAR_HOME/instances/python-instance/python_instance_
 PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"}
 DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR=$PULSAR_HOME/instances/deps
 FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR}"}
-SQL_HOME=$PULSAR_HOME/pulsar-sql
-TRINO_HOME=${PULSAR_HOME}/trino
-DEFAULT_PULSAR_TRINO_CONF=${TRINO_HOME}/conf
 
 pulsar_help() {
     cat <<EOF
@@ -60,8 +57,6 @@ where command is one of:
     proxy               Run a pulsar proxy
     websocket           Run a web socket proxy server
     functions-worker    Run a functions worker server
-    sql-worker          Run a sql worker server
-    sql                 Run sql CLI
     standalone          Run a broker server with local bookies and local zookeeper
     autorecovery        Run an autorecovery service
 
@@ -90,7 +85,6 @@ Environment variables:
    PULSAR_PROXY_CONF             Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF)
    PULSAR_WORKER_CONF            Configuration file for functions worker (default: $DEFAULT_WORKER_CONF)
    PULSAR_STANDALONE_CONF        Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF)
-   PULSAR_TRINO_CONF             Configuration directory for Pulsar SQL (default: $DEFAULT_PULSAR_TRINO_CONF)
    PULSAR_EXTRA_OPTS             Extra options to be passed to the jvm
    PULSAR_EXTRA_CLASSPATH        Add extra paths to the pulsar classpath
    PULSAR_PID_DIR                Folder where the pulsar server PID file should be stored
@@ -179,20 +173,6 @@ if [ ! -f "${PY_INSTANCE_FILE}" ]; then
     PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE}
 fi
 
-# find pulsar sql trino distribution location
-check_trino_libraries() {
-    if [ ! -d "${TRINO_HOME}" ]; then
-        BUILT_TRINO_HOME="${SQL_HOME}/presto-distribution/target/pulsar-presto-distribution"
-        if [ ! -d "${BUILT_TRINO_HOME}" ]; then
-            echo "\nCouldn't find trino distribution.";
-            echo "Make sure you've run 'mvn package'\n";
-            exit 1;
-        fi
-        TRINO_HOME=${BUILT_TRINO_HOME}
-        PULSAR_TRINO_CONF=${BUILT_TRINO_HOME}/conf
-    fi
-}
-
 add_maven_deps_to_classpath() {
     MVN="mvn"
     if [ "$MAVEN_HOME" != "" ]; then
@@ -259,21 +239,6 @@ if [ -z "$PULSAR_LOG_CONF" ]; then
     PULSAR_LOG_CONF=$DEFAULT_LOG_CONF
 fi
 
-if [ -z "$PULSAR_TRINO_CONF" ]; then
-    # TODO: As PIP-200 accepted, this compatibility is not promised. Refactor when we drop this b/w compatibility.
-    if [ -z "$PULSAR_PRESTO_CONF" ]; then
-      PULSAR_TRINO_CONF=$DEFAULT_PULSAR_TRINO_CONF
-    else
-      PULSAR_TRINO_CONF=$PULSAR_PRESTO_CONF
-    fi
-    if [ ! -d "${PULSAR_TRINO_CONF}" ]; then
-      FALLBACK_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
-      if [ -d "${FALLBACK_PULSAR_PRESTO_CONF}" ]; then
-        PULSAR_TRINO_CONF=$FALLBACK_PULSAR_PRESTO_CONF
-      fi
-    fi
-fi
-
 if [ -z "$FUNCTIONS_LOG_CONF" ]; then
   FUNCTIONS_LOG_CONF=$DEFAULT_FUNCTIONS_LOG_CONF
 fi
@@ -410,12 +375,6 @@ elif [ $COMMAND == "broker-tool" ]; then
     exec $JAVA $OPTS org.apache.pulsar.broker.tools.BrokerTool $@
 elif [ $COMMAND == "compact-topic" ]; then
     exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF $@
-elif [ $COMMAND == "sql" ]; then
-    check_trino_libraries
-    exec $JAVA -cp "${TRINO_HOME}/lib/*" io.trino.cli.Trino --server localhost:8081 "${@}"
-elif [ $COMMAND == "sql-worker" ]; then
-    check_trino_libraries
-    exec python3 ${TRINO_HOME}/bin/launcher.py --etc-dir ${PULSAR_TRINO_CONF} "${@}"
 elif [ $COMMAND == "tokens" ]; then
     exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils $@
 elif [ $COMMAND == "version" ]; then

bin/pulsar

@@ -160,7 +160,7 @@ function test_group_proxy() {
 function test_group_other() {
   mvn_test --clean --install \
            -pl '!org.apache.pulsar:distribution,!org.apache.pulsar:pulsar-offloader-distribution,!org.apache.pulsar:pulsar-server-distribution,!org.apache.pulsar:pulsar-io-distribution,!org.apache.pulsar:pulsar-all-docker-image' \
-           -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DdisableSqlMainProfile=true -DskipIntegrationTests \
+           -PskipTestsForUnitGroupOther -DdisableIoMainProfile=true -DskipIntegrationTests \
            -Dexclude='**/ManagedLedgerTest.java,
                    **/OffloadersCacheTest.java
                   **/PrimitiveSchemaTest.java,
@@ -175,7 +175,7 @@ function test_group_other() {
 
   echo "::endgroup::"
   local modules_with_quarantined_tests=$(git grep -l '@Test.*"quarantine"' | grep '/src/test/java/' | \
-    awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy|pulsar-io|pulsar-sql|pulsar-client' | sort | uniq | \
+    awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy|pulsar-io|pulsar-client' | sort | uniq | \
     perl -0777 -p -e 's/\n(\S)/,$1/g')
   if [ -n "${modules_with_quarantined_tests}" ]; then
     echo "::group::Running quarantined tests outside of pulsar-broker & pulsar-proxy (if any)"
@@ -190,10 +190,6 @@ function test_group_pulsar_io() {
     echo "::group::Running pulsar-io tests"
     mvn_test --install -Ppulsar-io-tests,-main
     echo "::endgroup::"
-
-    echo "::group::Running pulsar-sql tests"
-    mvn_test --install -Ppulsar-sql-tests,-main -DtestForkCount=1
-    echo "::endgroup::"
 }
 
 function test_group_pulsar_io_elastic() {

build/run_unit_group.sh

@@ -378,22 +378,6 @@
       <activation>
         <activeByDefault>true</activeByDefault>
       </activation>
-      <!-- depend on pulsar-presto-distribution by default, unless core-modules profile is active -->
-      <dependencies>
-        <dependency>
-          <groupId>${project.groupId}</groupId>
-          <artifactId>pulsar-presto-distribution</artifactId>
-          <version>${project.version}</version>
-          <type>tar.gz</type>
-          <scope>provided</scope>
-          <exclusions>
-            <exclusion>
-              <groupId>*</groupId>
-              <artifactId>*</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-      </dependencies>
     </profile>
   </profiles>
 </project>

distribution/server/pom.xml

@@ -60,22 +60,6 @@
       <directory>${basedir}/../../pulsar-functions/python-examples</directory>
       <outputDirectory>examples/python-examples</outputDirectory>
     </fileSet>
-    <fileSet>
-      <directory>${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution</directory>
-      <outputDirectory>trino</outputDirectory>
-      <excludes>
-        <exclude>bin</exclude>
-        <exclude>bin/**</exclude>
-      </excludes>
-    </fileSet>
-    <fileSet>
-      <directory>${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution</directory>
-      <outputDirectory>trino</outputDirectory>
-      <includes>
-        <include>bin/**</include>
-      </includes>
-      <fileMode>755</fileMode>
-    </fileSet>
   </fileSets>
   <files>
     <file>

distribution/server/src/assemble/bin.xml

@@ -368,8 +368,7 @@ services:
     image: apachepulsar/pulsar-all:latest
     restart: on-failure
     command: >
-      bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \
-               bin/apply-config-from-env.py conf/pulsar_env.sh && \
+      bash -c "bin/apply-config-from-env.py conf/pulsar_env.sh && \
                bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \
                exec bin/pulsar sql-worker run"
     environment:

docker-compose/kitchen-sink/docker-compose.yml

@@ -42,10 +42,6 @@ RUN for SUBDIRECTORY in conf data download logs; do \
      chmod -R g+w /pulsar/$SUBDIRECTORY; \
      done
 
-# Trino writes logs to this directory (at least during tests), so we need to give the process permission
-# to create those log directories. This should be removed when Trino is removed.
-RUN chmod g+w /pulsar/trino
-
 ### Create 2nd stage from Ubuntu image
 ### and add OpenJDK and Python dependencies (for Pulsar functions)
 

docker/pulsar/Dockerfile

@@ -53,7 +53,6 @@ static Pair<NarClassLoader, LedgerOffloaderFactory> getOffloaderFactory(String n
             throws IOException {
         // need to load offloader NAR to the classloader that also loaded LedgerOffloaderFactory in case
         // LedgerOffloaderFactory is loaded by a classloader that is not the default classloader
-        // as is the case for the pulsar presto plugin
         NarClassLoader ncl = NarClassLoaderBuilder.builder()
                 .narFile(new File(narPath))
                 .parentClassLoader(LedgerOffloaderFactory.class.getClassLoader())

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/offload/OffloaderUtils.java

@@ -191,7 +191,6 @@ flexible messaging model and an intuitive client API.</description>
     <json-smart.version>2.4.10</json-smart.version>
     <opensearch.version>1.2.4</opensearch.version>
     <elasticsearch-java.version>8.5.2</elasticsearch-java.version>
-    <trino.version>368</trino.version>
     <debezium.version>1.9.7.Final</debezium.version>
     <debezium.postgresql.version>42.5.0</debezium.postgresql.version>
     <debezium.mysql.version>8.0.30</debezium.mysql.version>
@@ -1776,7 +1775,6 @@ flexible messaging model and an intuitive client API.</description>
 
             <!-- This is a text property file that contains just a class name -->
             <exclude>**/META-INF/services/com.scurrilous.circe.HashProvider</exclude>
-            <exclude>**/META-INF/services/io.trino.spi.Plugin</exclude>
 
             <!-- Django generated code -->
             <exclude>**/django/stats/migrations/*.py</exclude>
@@ -2214,7 +2212,6 @@ flexible messaging model and an intuitive client API.</description>
         <module>pulsar-testclient</module>
         <module>pulsar-broker-auth-athenz</module>
         <module>pulsar-client-auth-athenz</module>
-        <module>pulsar-sql</module>
         <module>pulsar-broker-auth-oidc</module>
         <module>pulsar-broker-auth-sasl</module>
         <module>pulsar-client-auth-sasl</module>
@@ -2500,13 +2497,6 @@ flexible messaging model and an intuitive client API.</description>
       </modules>
     </profile>
 
-    <profile>
-      <id>pulsar-sql-tests</id>
-      <modules>
-        <module>pulsar-sql</module>
-      </modules>
-    </profile>
-
     <profile>
       <id>microbench</id>
       <modules>