From 480dfbf2733865c3129326da53d616b2a0acc233 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Sun, 4 Jan 2026 10:17:46 +0800 Subject: [PATCH 1/2] CI: Add support for Ubuntu 24.04 Docker and DEB builds This commit introduces comprehensive support for Ubuntu 24.04 (Noble Numbat) across build environments and packaging metadata. Key changes and package updates for Ubuntu 24.04: - Compiler Upgrade: Migrated from GCC/G++ 11 to GCC/G++ 13 to align with Noble's default toolchain. - Python 3.12 Migration: Updated system Python to 3.12. Removed python3-distutils as it has been deprecated and removed from Ubuntu 24.04 repositories (PEP 632). - t64 Transition: Updated DEB runtime dependencies to include the 't64' suffix (e.g., libssl3t64, libapr1t64, libcurl4t64) to comply with Noble's mandatory 64-bit time_t ABI transition. - libcgroup Update: Switched from libcgroup1 to libcgroup2 to match the updated library names in Ubuntu 24.04. - PIP Compliance: Added --break-system-packages flag for PIP installations within the Dockerfile to satisfy PEP 668 requirements. --- .../docker-cbdb-build-containers.yml | 7 +- .../workflows/docker-cbdb-test-containers.yml | 7 +- .../build/packaging/deb/ubuntu24.04/changelog | 6 + devops/build/packaging/deb/ubuntu24.04/compat | 2 + .../build/packaging/deb/ubuntu24.04/control | 107 +++++++++ .../build/packaging/deb/ubuntu24.04/install | 2 + .../build/packaging/deb/ubuntu24.04/postinst | 13 + devops/build/packaging/deb/ubuntu24.04/rules | 49 ++++ .../packaging/deb/ubuntu24.04/source/format | 2 + .../deb/ubuntu24.04/source/local-options | 3 + .../docker/build/ubuntu24.04/Dockerfile | 200 ++++++++++++++++ .../build/ubuntu24.04/configs/90-cbdb-limits | 32 +++ .../ubuntu24.04/configs/gpinitsystem.conf | 90 +++++++ .../build/ubuntu24.04/configs/init_system.sh | 195 +++++++++++++++ .../tests/testinfra/test_cloudberry_db_env.py | 128 ++++++++++ .../deploy/docker/test/ubuntu24.04/Dockerfile | 138 +++++++++++ .../test/ubuntu24.04/configs/90-cbdb-limits | 33 +++ .../ubuntu24.04/configs/gpinitsystem.conf | 88 +++++++ .../test/ubuntu24.04/configs/init_system.sh | 225 ++++++++++++++++++ 19 files changed, 1323 insertions(+), 4 deletions(-) create mode 100644 devops/build/packaging/deb/ubuntu24.04/changelog create mode 100644 devops/build/packaging/deb/ubuntu24.04/compat create mode 100644 devops/build/packaging/deb/ubuntu24.04/control create mode 100644 devops/build/packaging/deb/ubuntu24.04/install create mode 100644 devops/build/packaging/deb/ubuntu24.04/postinst create mode 100644 devops/build/packaging/deb/ubuntu24.04/rules create mode 100644 devops/build/packaging/deb/ubuntu24.04/source/format create mode 100644 devops/build/packaging/deb/ubuntu24.04/source/local-options create mode 100644 devops/deploy/docker/build/ubuntu24.04/Dockerfile create mode 100644 devops/deploy/docker/build/ubuntu24.04/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/build/ubuntu24.04/configs/gpinitsystem.conf create mode 100644 devops/deploy/docker/build/ubuntu24.04/configs/init_system.sh create mode 100644 devops/deploy/docker/build/ubuntu24.04/tests/testinfra/test_cloudberry_db_env.py create mode 100644 devops/deploy/docker/test/ubuntu24.04/Dockerfile create mode 100644 devops/deploy/docker/test/ubuntu24.04/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/test/ubuntu24.04/configs/gpinitsystem.conf create mode 100644 devops/deploy/docker/test/ubuntu24.04/configs/init_system.sh diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 1b13e9ff3f4..dd9ea9acd27 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -61,6 +61,7 @@ on: - 'devops/deploy/docker/build/rocky8/**' - 'devops/deploy/docker/build/rocky9/**' - 'devops/deploy/docker/build/ubuntu22.04/**' + - 'devops/deploy/docker/build/ubuntu24.04/**' pull_request: paths: - 'devops/deploy/docker/build/**' @@ -77,10 +78,10 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest - # Matrix strategy to build for both Rocky Linux 8 and 9, Ubuntu 22.04 + # Matrix strategy to build for both Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 strategy: matrix: - platform: ['rocky8', 'rocky9', 'ubuntu22.04'] + platform: ['rocky8', 'rocky9', 'ubuntu22.04', 'ubuntu24.04'] steps: # Checkout repository code with full history @@ -109,6 +110,8 @@ jobs: - 'devops/deploy/docker/build/rocky9/**' ubuntu22.04: - 'devops/deploy/docker/build/ubuntu22.04/**' + ubuntu24.04: + - 'devops/deploy/docker/build/ubuntu24.04/**' # Set up QEMU for multi-architecture support # This allows building ARM64 images on AMD64 infrastructure and vice versa diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index fcee6fa41b6..1c8e1c8a9a2 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -50,6 +50,7 @@ on: - 'devops/deploy/docker/test/rocky8/**' - 'devops/deploy/docker/test/rocky9/**' - 'devops/deploy/docker/test/ubuntu22.04/**' + - 'devops/deploy/docker/test/ubuntu24.04/**' pull_request: paths: - 'devops/deploy/docker/test/**' @@ -66,8 +67,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Build for both Rocky Linux 8 and 9, Ubuntu 22.04 - platform: ['rocky8', 'rocky9', 'ubuntu22.04'] + # Build for Rocky Linux 8 and 9, Ubuntu 22.04 and 24.04 + platform: ['rocky8', 'rocky9', 'ubuntu22.04', 'ubuntu24.04'] steps: # Checkout repository code @@ -93,6 +94,8 @@ jobs: - 'devops/deploy/docker/test/rocky9/**' ubuntu22.04: - 'devops/deploy/docker/test/ubuntu22.04/**' + ubuntu24.04: + - 'devops/deploy/docker/test/ubuntu24.04/**' # Skip if no changes for current platform - name: Skip if not relevant diff --git a/devops/build/packaging/deb/ubuntu24.04/changelog b/devops/build/packaging/deb/ubuntu24.04/changelog new file mode 100644 index 00000000000..f78f361e5d3 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/changelog @@ -0,0 +1,6 @@ +apache-cloudberry-db-incubating (2.0.0) noble; urgency=medium + + * Initial release. + + -- Cloudberry Team Wed, 26 Mar 2025 11:10:44 +0000 + diff --git a/devops/build/packaging/deb/ubuntu24.04/compat b/devops/build/packaging/deb/ubuntu24.04/compat new file mode 100644 index 00000000000..48c962ffa2b --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/compat @@ -0,0 +1,2 @@ +9 + diff --git a/devops/build/packaging/deb/ubuntu24.04/control b/devops/build/packaging/deb/ubuntu24.04/control new file mode 100644 index 00000000000..a561d8a4386 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/control @@ -0,0 +1,107 @@ +Source: apache-cloudberry-db-incubating +Maintainer: Apache Cloudberry (Incubating) +Section: database +Build-Depends: debhelper (>= 9), + bison, + ca-certificates-java, + ca-certificates, + cmake, + curl, + cgroup-tools, + flex, + gcc-13, + g++-13, + git, + krb5-multidev, + libapr1-dev, + libbz2-dev, + libcurl4-gnutls-dev, + libevent-dev, + libkrb5-dev, + libldap-dev, + libperl-dev, + libreadline-dev, + libssl-dev, + libxml2-dev, + libyaml-dev, + libzstd-dev, + libaprutil1-dev, + libpam0g-dev, + libpam0g, + libcgroup2, + libyaml-0-2, + libldap2, + libssl3t64, + libxerces-c-dev, + libxerces-c3.2t64, + ninja-build, + quilt, + unzip, + wget, + zlib1g-dev, + libuv1-dev + +Package: apache-cloudberry-db-incubating +Provides: apache-cloudberry-db +Architecture: amd64 +Depends: curl, + cgroup-tools, + iputils-ping, + iproute2, + keyutils, + krb5-multidev, + less, + libapr1t64, + libbz2-1.0, + libcurl4t64, + libcurl3t64-gnutls, + libevent-2.1-7t64, + libreadline8t64, + libxml2, + libyaml-0-2, + libldap2, + libzstd1, + libcgroup2, + libssl3t64, + libpam0g, + libprotobuf32t64, + libpsl5t64, + libuv1t64, + liburing2, + libxerces-c3.2t64, + locales, + lsof, + lz4, + net-tools, + openssh-client, + openssh-server, + openssl, + python3, + rsync, + wget, + xz-utils, + zlib1g +Description: Apache Cloudberry (incubating) is an advanced, open-source, massively + parallel processing (MPP) data warehouse developed from PostgreSQL and + Greenplum. It is designed for high-performance analytics on + large-scale data sets, offering powerful analytical capabilities and + enhanced security features. + Key Features: + - Massively parallel processing for optimized performance + - Advanced analytics for complex data processing + - Integration with ETL and BI tools + - Compatibility with multiple data sources and formats + - Enhanced security features + Apache Cloudberry supports both batch processing and real-time data + warehousing, making it a versatile solution for modern data + environments. + Apache Cloudberry (incubating) is an effort undergoing incubation at + the Apache Software Foundation (ASF), sponsored by the Apache + Incubator PMC. + Incubation is required of all newly accepted projects until a further + review indicates that the infrastructure, communications, and decision + making process have stabilized in a manner consistent with other + successful ASF projects. + While incubation status is not necessarily a reflection of the + completeness or stability of the code, it does indicate that the + project has yet to be fully endorsed by the ASF. diff --git a/devops/build/packaging/deb/ubuntu24.04/install b/devops/build/packaging/deb/ubuntu24.04/install new file mode 100644 index 00000000000..e3d847a3c99 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/install @@ -0,0 +1,2 @@ +debian/build/* /usr/cloudberry-db + diff --git a/devops/build/packaging/deb/ubuntu24.04/postinst b/devops/build/packaging/deb/ubuntu24.04/postinst new file mode 100644 index 00000000000..708e3ed29b7 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/postinst @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin +GPHOME=/usr/cloudberry-db + +if [ "$1" = configure ]; then + + ln -sf ${GPHOME} /usr/local/cloudberry-db + +fi + diff --git a/devops/build/packaging/deb/ubuntu24.04/rules b/devops/build/packaging/deb/ubuntu24.04/rules new file mode 100644 index 00000000000..6213985b48c --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/rules @@ -0,0 +1,49 @@ +#!/usr/bin/make -f + +DH_VERBOSE = 1 +DPKG_EXPORT_BUILDFLAGS = 1 + +CBDB_BIN_PATH := /usr/local/cloudberry-db +DEBIAN_DESTINATION := $(shell pwd)/debian/build + +# assumes that CWD is root of cbdb source +CBDB_PKG_VERSION := $(CBDB_PKG_VERSION) +PACKAGE_CBDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) +PATH := ${DEBIAN_DESTINATION}/bin:${PATH} + +.PHONY: gpinstall + +include /usr/share/dpkg/default.mk + +%: + dh $@ --parallel + +gpinstall: + make install + +override_dh_auto_install: gpinstall + # the staging directory for creating a debian is NOT the right GPHOME. + # change GPHOME to point to the post-install target install directory. + sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/cloudberry-env.sh + +override_dh_auto_build: + echo "Skipping build" + +override_dh_auto_clean: + echo "Skipping clean" + +override_dh_auto_configure: + echo "Skipping configure" + +override_dh_auto_test: + echo "Skipping auto test" + +override_dh_gencontrol: + echo "using version ${CBDB_PKG_VERSION} for binary Cloudberry" + dh_gencontrol -- -v${CBDB_PKG_VERSION} -p${PACKAGE_CBDB} + +override_dh_shlibdeps: + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/libfakeroot:${DEBIAN_DESTINATION}/lib dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + +clean_dev_local: + rm -rf ${DEBIAN_DESTINATION} diff --git a/devops/build/packaging/deb/ubuntu24.04/source/format b/devops/build/packaging/deb/ubuntu24.04/source/format new file mode 100644 index 00000000000..435dd71905d --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/source/format @@ -0,0 +1,2 @@ +3.0 (native) + diff --git a/devops/build/packaging/deb/ubuntu24.04/source/local-options b/devops/build/packaging/deb/ubuntu24.04/source/local-options new file mode 100644 index 00000000000..adf39a30992 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu24.04/source/local-options @@ -0,0 +1,3 @@ +#abort-on-upstream-changes +#unapply-patches + diff --git a/devops/deploy/docker/build/ubuntu24.04/Dockerfile b/devops/deploy/docker/build/ubuntu24.04/Dockerfile new file mode 100644 index 00000000000..c4f4e646720 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu24.04/Dockerfile @@ -0,0 +1,200 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu noble 24.04 -based container to serve as +# a base environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +FROM ubuntu:24.04 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} +ENV DEBIAN_FRONTEND=noninteractive + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- + +RUN apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget && \ + apt-get install -y -qq locales && \ + locale-gen "en_US.UTF-8" && \ + update-locale LANG="en_US.UTF-8" && \ + apt-get install -y -qq \ + bison \ + build-essential \ + cmake \ + dpkg-dev \ + fakeroot \ + flex \ + g++-13 \ + gcc-13 \ + gdb \ + git \ + iproute2 \ + iputils-ping \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libipc-run-perl \ + libkrb5-dev \ + libldap-dev \ + liblz4-dev \ + libpam0g-dev \ + libperl-dev \ + libprotobuf-dev \ + libreadline-dev \ + libssh2-1-dev \ + libssl-dev \ + liburing-dev \ + libuv1-dev \ + libxerces-c-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + lsof \ + make \ + openssh-server \ + pkg-config \ + protobuf-compiler \ + python3-pip \ + python3-setuptools \ + python3.12 \ + python3.12-dev \ + python3-wheel \ + rsync \ + sudo \ + tzdata \ + zlib1g-dev && \ + apt-get install -y -qq \ + ca-certificates-java \ + cgroup-tools \ + curl \ + debhelper \ + libaprutil1-dev \ + libcgroup2 \ + ninja-build \ + quilt \ + unzip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + cd && GO_VERSION="go1.23.4" && \ + ARCH=$(uname -m) && \ + if [ "${ARCH}" = "aarch64" ]; then \ + GO_ARCH="arm64" && \ + GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + elif [ "${ARCH}" = "x86_64" ]; then \ + GO_ARCH="amd64" && \ + GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + else \ + echo "Unsupported architecture: ${ARCH}" && exit 1; \ + fi && \ + GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + wget -nv "${GO_URL}" && \ + echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - && \ + tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + mv go "/usr/local/${GO_VERSION}" && \ + ln -s "/usr/local/${GO_VERSION}" /usr/local/go && \ + rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/gcc-13 100 && \ + update-alternatives --set gcc /usr/bin/gcc-13 && \ + update-alternatives --set g++ /usr/bin/g++-13 + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- + +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + chmod 755 /tmp/init_system.sh && \ + /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin -s /bin/bash && \ + echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + ssh-keygen -A && \ + mkdir /var/run/sshd && chmod 0755 /var/run/sshd + +# Install testinfra via pip +RUN pip3 install --break-system-packages pytest-testinfra + +# Example: Copying test files into the container +COPY tests /tests + +USER gpadmin +WORKDIR /home/gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] + diff --git a/devops/deploy/docker/build/ubuntu24.04/configs/90-cbdb-limits b/devops/deploy/docker/build/ubuntu24.04/configs/90-cbdb-limits new file mode 100644 index 00000000000..8ea1d9ed988 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu24.04/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits + # -------------------------------------------------------------------- + # + # Licensed to the Apache Software Foundation (ASF) under one or more + # contributor license agreements. See the NOTICE file distributed + # with this work for additional information regarding copyright + # ownership. The ASF licenses this file to You under the Apache + # License, Version 2.0 (the "License"); you may not use this file + # except in compliance with the License. You may obtain a copy of the + # License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + # implied. See the License for the specific language governing + # permissions and limitations under the License. + # + # -------------------------------------------------------------------- + + # Core dump file size limits for gpadmin + gpadmin soft core unlimited + gpadmin hard core unlimited + + # Open file limits for gpadmin + gpadmin soft nofile 524288 + gpadmin hard nofile 524288 + + # Process limits for gpadmin + gpadmin soft nproc 131072 + gpadmin hard nproc 131072 diff --git a/devops/deploy/docker/build/ubuntu24.04/configs/gpinitsystem.conf b/devops/deploy/docker/build/ubuntu24.04/configs/gpinitsystem.conf new file mode 100644 index 00000000000..282c21762ab --- /dev/null +++ b/devops/deploy/docker/build/ubuntu24.04/configs/gpinitsystem.conf @@ -0,0 +1,90 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# -------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# -------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# -------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# -------------------------------------------------------------------- + diff --git a/devops/deploy/docker/build/ubuntu24.04/configs/init_system.sh b/devops/deploy/docker/build/ubuntu24.04/configs/init_system.sh new file mode 100644 index 00000000000..52a44462c61 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu24.04/configs/init_system.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +## Container Initialization Script +# -------------------------------------------------------------------- +## This script sets up the environment inside the Docker container for +## the Apache Cloudberry Build Environment. It performs the following +## tasks: +## +## 1. Verifies that the container is running with the expected hostname. +## 2. Starts the SSH daemon to allow SSH access to the container. +## 3. Configures passwordless SSH access for the 'gpadmin' user. +## 4. Displays a welcome banner and system information. +## 5. Starts an interactive bash shell. +## +## This script is intended to be used as an entrypoint or initialization +## script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +cat <<-EOF +Welcome to the Apache Cloudberry Build Environment! + +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# Add go to PATH +source /etc/profile.d/go.sh + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/devops/deploy/docker/build/ubuntu24.04/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/ubuntu24.04/tests/testinfra/test_cloudberry_db_env.py new file mode 100644 index 00000000000..2262350ece2 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu24.04/tests/testinfra/test_cloudberry_db_env.py @@ -0,0 +1,128 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +import testinfra + +def test_installed_packages(host): + """ + Test if the essential packages are installed. + """ + packages = [ + "bat", + "bison", + "cmake", + "flex", + "g++-13", + "gcc-13", + "git", + "htop", + "iproute2", + "iputils-ping", + "libapr1-dev", + "libbz2-dev", + "libcurl4-gnutls-dev", + "libevent-dev", + "libipc-run-perl", + "libkrb5-dev", + "libldap-dev", + "liblz4-dev", + "libpam0g-dev", + "libperl-dev", + "libprotobuf-dev", + "libreadline-dev", + "libssl-dev", + "libuv1-dev", + "libxerces-c-dev", + "libxml2-dev", + "libyaml-dev", + "libzstd-dev", + "locales", + "lsof", + "make", + "openssh-server", + "pkg-config", + "protobuf-compiler", + "python3-pip", + "python3-setuptools", + "python3.12", + "python3.12-dev", + "rsync", + "silversearcher-ag", + "sudo", + "tzdata", + "vim", + "wget", + "zlib1g-dev" + ] + for package in packages: + pkg = host.package(package) + assert pkg.is_installed + + +def test_user_gpadmin_exists(host): + """ + Test if the gpadmin user exists and is configured properly. + """ + user = host.user("gpadmin") + assert user.exists + assert "gpadmin" in user.groups + + +def test_ssh_service(host): + """ + Test if SSH service is configured correctly. + """ + sshd_config = host.file("/etc/ssh/sshd_config") + assert sshd_config.exists + + +def test_timezone(host): + """ + Test if the timezone is configured correctly. + """ + localtime = host.file("/etc/localtime") + assert localtime.exists + + +def test_system_limits_configured(host): + """ + Test if the custom system limits are applied. + """ + limits_file = host.file("/etc/security/limits.d/90-cbdb-limits") + assert limits_file.exists + + +def test_init_system_script(host): + """ + Test if the init_system.sh script is present and executable. + """ + script = host.file("/tmp/init_system.sh") + assert script.exists + assert script.mode == 0o755 + + +def test_locale_generated(host): + """ + Test if the en_US.UTF-8 locale is correctly generated. + """ + locale = host.run("locale -a | grep en_US.utf8") + assert locale.exit_status == 0 + assert "en_US.utf8" in locale.stdout + diff --git a/devops/deploy/docker/test/ubuntu24.04/Dockerfile b/devops/deploy/docker/test/ubuntu24.04/Dockerfile new file mode 100644 index 00000000000..55886282065 --- /dev/null +++ b/devops/deploy/docker/test/ubuntu24.04/Dockerfile @@ -0,0 +1,138 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Test Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu noble 24.04 -based container to serve as +# a test environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-test-env . +# docker run -h cdw -it cloudberry-db-test-env +# -------------------------------------------------------------------- + +FROM ubuntu:24.04 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NOWARNINGS="yes" + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- + +RUN apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget \ + gdb \ + git \ + iputils-ping \ + lsof \ + openssh-server \ + pkg-config \ + python3.12 \ + python3-pip \ + python3-setuptools \ + sudo \ + tzdata && \ + apt-get install -y -qq locales && \ + locale-gen ${LANG} && \ + update-locale LANG=${LANG} && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/python3.12 /usr/bin/python + +# -------------------------------------------------------------------- +# User Creation and Configuration +# -------------------------------------------------------------------- +# - Create the 'gpadmin' user and group. +# - Configure the 'gpadmin' user with passwordless sudo privileges. +# - Add Cloudberry-specific entries to the gpadmin's .bashrc. +# -------------------------------------------------------------------- +RUN /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin -s /bin/bash && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + echo '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + chmod 755 /tmp/init_system.sh && \ + ssh-keygen -A + +USER gpadmin +WORKDIR /home/gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] + diff --git a/devops/deploy/docker/test/ubuntu24.04/configs/90-cbdb-limits b/devops/deploy/docker/test/ubuntu24.04/configs/90-cbdb-limits new file mode 100644 index 00000000000..c53e9bd1942 --- /dev/null +++ b/devops/deploy/docker/test/ubuntu24.04/configs/90-cbdb-limits @@ -0,0 +1,33 @@ +# /etc/security/limits.d/90-db-limits +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 + diff --git a/devops/deploy/docker/test/ubuntu24.04/configs/gpinitsystem.conf b/devops/deploy/docker/test/ubuntu24.04/configs/gpinitsystem.conf new file mode 100644 index 00000000000..bd01e52c505 --- /dev/null +++ b/devops/deploy/docker/test/ubuntu24.04/configs/gpinitsystem.conf @@ -0,0 +1,88 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# ---------------------------------------------------------------------- +# gpinitsystem Configuration File for Cloudberry Database +# ---------------------------------------------------------------------- +# This configuration file is used to initialize a Cloudberry Database +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# ---------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# ---------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# ---------------------------------------------------------------------- + diff --git a/devops/deploy/docker/test/ubuntu24.04/configs/init_system.sh b/devops/deploy/docker/test/ubuntu24.04/configs/init_system.sh new file mode 100644 index 00000000000..09eadc2b736 --- /dev/null +++ b/devops/deploy/docker/test/ubuntu24.04/configs/init_system.sh @@ -0,0 +1,225 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Container Initialization Script +# -------------------------------------------------------------------- +# This script sets up the environment inside the Docker container for +# the Apache Cloudberry Build Environment. It performs the following +# tasks: +# +# 1. Verifies that the container is running with the expected hostname. +# 2. Starts the SSH daemon to allow SSH access to the container. +# 3. Configures passwordless SSH access for the 'gpadmin' user. +# 4. Sets up the necessary directories and configuration files for +# Apache Cloudberry. +# 5. Displays a welcome banner and system information. +# 6. Starts an interactive bash shell. +# +# This script is intended to be used as an entrypoint or initialization +# script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +sudo mkdir -p /run/sshd +sudo chmod 755 /run/sshd + +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# -------------------------------------------------------------------- +# Cloudberry Data Directories Setup +# -------------------------------------------------------------------- +# The script sets up the necessary directories for Apache Cloudberry, +# including directories for the coordinator, standby coordinator, primary +# segments, and mirror segments. It also sets up the configuration files +# required for initializing the database. +# -------------------------------------------------------------------- +sudo rm -rf /data1/* +sudo mkdir -p /data1/coordinator /data1/standby_coordinator /data1/primary /data1/mirror +sudo chown -R gpadmin.gpadmin /data1 + +# Copy the gpinitsystem configuration file to the home directory +cp /tmp/gpinitsystem.conf /home/gpadmin + +# Set up the hostfile for cluster initialization +echo $(hostname) > /home/gpadmin/hostfile_gpinitsystem + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# - Cloudberry version (if installed) +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +# Check if Apache Cloudberry is installed and display its version +if dpkg -l apache-cloudberry-db-incubating > /dev/null 2>&1; then + CBDB_VERSION=$(/usr/local/cbdb/bin/postgres --gp-version) +else + CBDB_VERSION="Not installed" +fi + +cat <<-EOF +Welcome to the Apache Cloudberry Test Environment! + +Cloudberry version .. : $CBDB_VERSION +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash + From c08991c2a64a99b10720c9d43232b370af996933 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 26 Jan 2026 14:22:07 +0800 Subject: [PATCH 2/2] Add news to the pom.xml for license check --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6d33d7e9de5..02769c19a5f 100644 --- a/pom.xml +++ b/pom.xml @@ -1742,11 +1742,13 @@ code or new licensing patterns. devops/deploy/docker/build/rocky8/tests/requirements.txt devops/deploy/docker/build/rocky9/tests/requirements.txt - devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt + devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt + devops/deploy/docker/build/ubuntu24.04/tests/requirements.txt devops/build/packaging/deb/ubuntu22.04/** + devops/build/packaging/deb/ubuntu24.04/**