diff --git a/.github/workflows/check-16k-page-alignment.yml b/.github/workflows/check-16k-page-alignment.yml new file mode 100644 index 00000000000..c54742dd6a3 --- /dev/null +++ b/.github/workflows/check-16k-page-alignment.yml @@ -0,0 +1,46 @@ +name: Check 16 KB page alignment + +# Google Play requires every 64-bit native library in an upload to be linked +# for 16 KB memory pages: API 35+ uploads since 2025-11-01, and Wear OS uploads +# containing native code from 2026-09-15. +# See https://developer.android.com/guide/practices/page-sizes. +# +# Alignment is decided at link time, so nothing downstream can repair it -- not +# AGP's ZIP alignment, not bundletool, not the developer. A Codename One +# artifact that gets it wrong ships a broken app the developer cannot fix, and +# the breakage is invisible until Play review or a real 16 KB device: the app +# builds and runs on every 4 KB device in the office. +# +# Deliberately its own workflow rather than a step in pr.yml, whose paths filter +# would let a PR that only adds a native artifact skip the check. The scan is a +# ZIP listing plus an ELF header read, so running it on every PR costs nothing. + +permissions: + contents: read + +on: + workflow_dispatch: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + check-16k-page-alignment: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Unit-test the gate's classification boundaries + # Every case in here is one a review caught: something unreadable + # passing as compliant, or a valid artifact rejected by a rule that + # does not apply to it. Run before the scan, so a broken gate is + # reported as a broken gate rather than as a clean tree. + shell: bash + run: python3 scripts/test_check_16k_page_alignment.py + + - name: Scan tracked native artifacts for 16 KB page alignment + shell: bash + run: scripts/check-16k-page-alignment.py --verbose diff --git a/CLAUDE.md b/CLAUDE.md index 471fd35a416..d53dfe6d24c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -253,6 +253,50 @@ Platform-specific native code locations: - **Android**: Within Android port module (Java/Kotlin) - **JavaSE**: Within JavaSE port (Java) +#### Prebuilt Android `.so` files must be 16 KB page aligned + +Google Play requires every **64-bit** shared library in an upload to be linked +for 16 KB memory pages -- API 35+ uploads since 2025-11-01, Wear OS uploads +containing native code from 2026-09-15. Each `PT_LOAD` segment needs `p_align` +of `0x4000` or more (). + +Alignment is settled at **link** time. AGP aligns the ZIP entries and bundletool +reports the outcome, but nothing downstream can move a segment inside a prebuilt +`.so`, so a Codename One artifact that gets it wrong ships a broken app the +developer cannot fix from their own sources. And it is silent: the app builds +and runs on every 4 KB device, and only Play review or a real 16 KB device says +otherwise. + +Build any Android native artifact with **NDK r28 or newer**. r28 is the first +release that links this way by default, and the first whose prebuilt +`libc++_shared.so` and `libomp.so` are aligned -- those are copied out of the +NDK, so `-Wl,-z,max-page-size=16384` cannot fix them on r26/r27. + +Separately, the NDK's own floor has been API 21 since r26, so any AAR built +here needs `minSdk 21` and an entry in `PlatformFeatureCatalog` with +`.androidMinimumSdk(21)` to lift the application's default floor of 19 past the +manifest merger. A lower `minSdk` in the library is a claim the `.so` files do +not honour -- check `.note.android.ident`, which records the API level the +slice was really compiled against. + +`scripts/check-16k-page-alignment.py` gates this over every tracked archive and +loose `.so` in the tree, in its own workflow +(`.github/workflows/check-16k-page-alignment.yml`) rather than inside `pr.yml`, +whose `paths` filter would let a PR that only adds a native artifact skip it. +Containers and libraries are found by magic bytes, not by extension, so a +`.cn1lib` or any future container is covered. + +The rule reaches only 64-bit libraries under Android packaging. 32-bit slices +are exempt because 16 KB pages are a 64-bit feature, and a desktop `.so` -- +which a cn1lib can ship through `nativelinux`/`nativese` -- is correct at +0x1000. Integrity is not scoped that way: a `.so` that is not an ELF, or is +truncated, fails wherever it sits. + +```bash +scripts/check-16k-page-alignment.py # every tracked artifact +scripts/check-16k-page-alignment.py --verbose # list everything scanned +``` + #### ParparVM native names are checked, and getting one wrong is silent ParparVM encodes the **whole Java signature in the C function name**. For diff --git a/maven/cn1-ai-whisper/android-aar/README.md b/maven/cn1-ai-whisper/android-aar/README.md index 6241ebc54a5..111a8881c36 100644 --- a/maven/cn1-ai-whisper/android-aar/README.md +++ b/maven/cn1-ai-whisper/android-aar/README.md @@ -21,3 +21,63 @@ WHISPER_INCLUDE_DIR=/path/to/whisper.cpp/include \ `arm64-v8a/libwhisper.so`. The build script copies the resulting AAR to `../android/src/main/resources/cn1-ai-whisper-android.aar`, where the cn1lib packager and Android builder already know how to pick it up. + +## whisper.cpp version + +The checked-in AAR is built from whisper.cpp `v1.9.1-77-g0874de3e`: + +```sh +git clone https://github.com/ggml-org/whisper.cpp.git +git -C whisper.cpp checkout 0874de3e +``` + +Record the new commit here whenever the AAR is rebuilt against a different one. +Nothing in the tree derives it, and the only reason it was recoverable the last +time was that the version and the abbreviated hash happened to survive as +strings inside `libcn1aiwhisper.so`. + +## 16 KB memory pages + +Google Play requires every **64-bit** shared library in an upload to be linked +for 16 KB memory pages -- for API 35+ uploads since 2025-11-01, and for Wear OS +uploads containing native code from 2026-09-15. In ELF terms every `PT_LOAD` +segment needs `p_align` of `0x4000` or more. See +. + +This is decided at **link** time. AGP aligns the ZIP entries and bundletool +reports the outcome, but neither can move a segment inside a prebuilt `.so`, so +an AAR that gets this wrong cannot be repaired by the application that consumes +it -- and the failure is silent until Play review or a real 16 KB device, +because the app builds and runs on every 4 KB device. + +Two things keep it right, and both are needed: + +- `build.gradle` pins **NDK r28** (`28.2.13676358`). r28 is the first release + that links this way by default, and, more to the point, the first whose + prebuilt `libc++_shared.so` and `libomp.so` are aligned. Those two are copied + into the AAR from the NDK itself, so no flag set in this project can fix them + on r26/r27. +- `src/main/cpp/CMakeLists.txt` passes `-Wl,-z,max-page-size=16384` and + `-Wl,-z,common-page-size=16384`, which covers `libcn1aiwhisper.so` if this + project is ever driven by an older NDK or a standalone CMake invocation. + +The library also declares `minSdk 21` now. That is not a consequence of the NDK +bump: the NDK's own floor has been API 21 since r26, so every slice this project +has ever shipped carries API 21 in its `.note.android.ident` while the AAR +manifest claimed 19. The old value was a claim the native code never honoured, +and an API 19 install would have failed at `System.loadLibrary`. Codename One's +default `android.min_sdk_version` is 19, so `PlatformFeatureCatalog` raises an +application's floor to 21 when it references `com.codename1.ai.whisper`; +without that the manifest merger rejects the build. + +Verify a rebuild before committing it: + +```sh +scripts/check-16k-page-alignment.py --verbose +``` + +It runs on every PR as `.github/workflows/check-16k-page-alignment.yml` and +covers every native artifact in the tree, not just this one. It finds +containers and libraries by magic bytes rather than by extension, and applies +the alignment rule only to 64-bit libraries under Android packaging -- a +desktop `.so` shipped through `nativelinux`/`nativese` is correct at 0x1000. diff --git a/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/build.gradle b/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/build.gradle index fa04197623c..dd65950d514 100644 --- a/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/build.gradle +++ b/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/build.gradle @@ -9,10 +9,27 @@ def whisperIncludeDir = project.findProperty('whisperIncludeDir') ?: System.gete android { namespace 'com.codename1.ai.whisper.android' compileSdk 33 - ndkVersion '26.3.11579264' + // NDK r28 is the first release that links shared libraries with 16 KB + // aligned PT_LOAD segments by default, which Google Play requires of every + // 64-bit native library (API 35+ uploads since 2025-11-01, Wear OS from + // 2026-09-15). The requirement covers libc++_shared.so and libomp.so too, + // and those are NDK *prebuilts* -- no linker flag applied to this project + // can realign them, so moving the NDK forward is the only fix rather than + // one option among several. scripts/check-16k-page-alignment.py gates the + // staged AAR. + ndkVersion '28.2.13676358' defaultConfig { - minSdk 19 + // 21 is what the native code has always actually required: the NDK's + // own floor has been API 21 since r26 (meta/platforms.json "min": 21), + // so the previous `minSdk 19` was a manifest claim the .so files never + // honoured -- every shipped slice, 32-bit included, carries API 21 in + // its .note.android.ident. Declaring it makes the AAR honest instead + // of letting an API 19 install fail at System.loadLibrary. + // PlatformFeatureCatalog raises a consuming app's + // android.min_sdk_version to match, so the manifest merge still + // succeeds for a default (19) project. + minSdk 21 externalNativeBuild { cmake { abiFilters 'armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64' diff --git a/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/src/main/cpp/CMakeLists.txt b/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/src/main/cpp/CMakeLists.txt index ae338e8d5f9..2a245cd651c 100644 --- a/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/src/main/cpp/CMakeLists.txt +++ b/maven/cn1-ai-whisper/android-aar/cn1-ai-whisper-android/src/main/cpp/CMakeLists.txt @@ -1,6 +1,18 @@ cmake_minimum_required(VERSION 3.22.1) project(cn1_ai_whisper_android LANGUAGES C CXX) +# Google Play requires every 64-bit shared library in an upload to have +# PT_LOAD segments aligned to 16 KB (API 35+ uploads since 2025-11-01, Wear OS +# from 2026-09-15) -- see https://developer.android.com/guide/practices/page-sizes. +# NDK r28, which build.gradle pins, already does this by default; these flags +# are what keeps the result correct if this project is ever driven by an older +# NDK or a standalone CMake invocation, where the default is still 4 KB and the +# breakage is silent (the library loads fine on every 4 KB device). +# +# Set before any target is declared so it also reaches the whisper.cpp +# subproject added below. +add_link_options(-Wl,-z,max-page-size=16384 -Wl,-z,common-page-size=16384) + add_library(cn1aiwhisper SHARED native_whisper_jni.cpp) target_compile_features(cn1aiwhisper PRIVATE cxx_std_17) diff --git a/maven/cn1-ai-whisper/android/src/main/resources/cn1-ai-whisper-android.aar b/maven/cn1-ai-whisper/android/src/main/resources/cn1-ai-whisper-android.aar index 079583d115b..52795e73dad 100644 Binary files a/maven/cn1-ai-whisper/android/src/main/resources/cn1-ai-whisper-android.aar and b/maven/cn1-ai-whisper/android/src/main/resources/cn1-ai-whisper-android.aar differ diff --git a/maven/cn1-ai-whisper/common/src/test/java/com/codename1/ai/whisper/AndroidWhisperAarProjectTest.java b/maven/cn1-ai-whisper/common/src/test/java/com/codename1/ai/whisper/AndroidWhisperAarProjectTest.java index 1c11607a686..48bcf604245 100644 --- a/maven/cn1-ai-whisper/common/src/test/java/com/codename1/ai/whisper/AndroidWhisperAarProjectTest.java +++ b/maven/cn1-ai-whisper/common/src/test/java/com/codename1/ai/whisper/AndroidWhisperAarProjectTest.java @@ -1,10 +1,40 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ package com.codename1.ai.whisper; import org.junit.jupiter.api.Test; +import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -12,6 +42,16 @@ class AndroidWhisperAarProjectTest { + /** + * Minimum {@code p_align} Google Play requires of a {@code PT_LOAD} + * segment in a 64-bit shared library. + */ + private static final long REQUIRED_PAGE_ALIGNMENT = 0x4000L; + + private static final int ELFCLASS32 = 1; + private static final int ELFCLASS64 = 2; + private static final int PT_LOAD = 1; + @Test void androidJniBridgeExportsTimedSegmentsForAllAndroidAbis() throws Exception { String cpp = read("../android-aar/cn1-ai-whisper-android/src/main/cpp/native_whisper_jni.cpp"); @@ -22,7 +62,28 @@ void androidJniBridgeExportsTimedSegmentsForAllAndroidAbis() throws Exception { String gradle = read("../android-aar/cn1-ai-whisper-android/build.gradle"); assertTrue(gradle.contains("abiFilters 'armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'"), gradle); - assertTrue(gradle.contains("ndkVersion '26.3.11579264'"), gradle); + assertTrue(gradle.contains("ndkVersion '28.2.13676358'"), gradle); + } + + /** + * The NDK is what supplies 16 KB aligned {@code libc++_shared.so} and + * {@code libomp.so}: those are prebuilts copied into the AAR, so no linker + * flag set here can realign them and only r28 or newer is compliant. The + * CMake flags cover the one library this project links itself, for a + * standalone CMake run or an older NDK. + */ + @Test + void nativeBuildIsConfiguredForSixteenKilobytePages() throws Exception { + String gradle = read("../android-aar/cn1-ai-whisper-android/build.gradle"); + assertTrue(gradle.contains("ndkVersion '28.2.13676358'"), gradle); + // 21 is what the native code always required: the NDK floor has been + // API 21 since r26, so the old `minSdk 19` was a manifest claim the + // .so files never honoured. + assertTrue(gradle.contains("minSdk 21"), gradle); + + String cmake = read("../android-aar/cn1-ai-whisper-android/src/main/cpp/CMakeLists.txt"); + assertTrue(cmake.contains("-Wl,-z,max-page-size=16384"), cmake); + assertTrue(cmake.contains("-Wl,-z,common-page-size=16384"), cmake); } @Test @@ -42,6 +103,151 @@ void stagedAndroidAarContainsJniSlicesWhenPresent() throws Exception { } } + /** + * Every 64-bit library in the staged AAR has to be linked for 16 KB memory + * pages, which Google Play requires of API 35+ uploads since 2025-11-01 and + * of Wear OS uploads from 2026-09-15. Alignment is fixed at link time, so + * an application cannot repair a Codename One artifact that got it wrong; + * the app simply builds, installs on the 4 KB devices the developer owns, + * and is rejected or fails to load on a 16 KB one. + * + *

32-bit slices are deliberately not checked. 16 KB pages are a 64-bit + * feature and {@code armeabi-v7a}/{@code x86} never run on such a device. + * The same rule is enforced across the whole tree by + * {@code scripts/check-16k-page-alignment.py}.

+ */ + @Test + void stagedAndroidAarIsSixteenKilobytePageAlignedWhenPresent() throws Exception { + File aar = new File("../android/src/main/resources/cn1-ai-whisper-android.aar"); + if (!aar.isFile()) { + return; + } + List misaligned = new ArrayList(); + int checked = 0; + ZipFile zip = new ZipFile(aar); + try { + Enumeration entries = zip.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (entry.isDirectory() || !entry.getName().endsWith(".so")) { + continue; + } + byte[] elf = readFully(zip, entry); + long[] alignments = loadSegmentAlignments(entry.getName(), elf); + if (alignments == null) { + // A 32-bit slice, which the rule does not cover. + continue; + } + checked++; + for (long alignment : alignments) { + if (alignment < REQUIRED_PAGE_ALIGNMENT) { + misaligned.add(entry.getName() + " p_align 0x" + + Long.toHexString(alignment)); + break; + } + } + } + } finally { + zip.close(); + } + assertTrue(checked > 0, + "No 64-bit libraries found in the AAR; the arm64-v8a and x86_64 " + + "slices are required for a Play upload"); + assertTrue(misaligned.isEmpty(), + "Rebuild the AAR with NDK r28 or newer -- these 64-bit libraries " + + "are not aligned to 0x4000: " + misaligned); + } + + /** + * Reads the {@code p_align} of every {@code PT_LOAD} program header, + * returning null for a library that is genuinely 32-bit and so outside + * the rule. + * + *

Anything that does not parse fails the test rather than returning + * null. An entry this method could not read is one whose alignment nobody + * checked, which must not read the same as one checked and found + * compliant -- and a truncated 64-bit library still starts with the ELF + * magic, so treating unreadable bytes as "32-bit, exempt" would make the + * exemption reachable by corruption.

+ */ + private static long[] loadSegmentAlignments(String name, byte[] elf) { + assertTrue(elf.length >= 5 && elf[0] == 0x7f && elf[1] == 'E' + && elf[2] == 'L' && elf[3] == 'F', + name + " is not an ELF file"); + int elfClass = elf[4] & 0xff; + assertTrue(elfClass == ELFCLASS32 || elfClass == ELFCLASS64, + name + " has unknown ELF class " + elfClass); + boolean is64 = elfClass == ELFCLASS64; + + // The two program header layouts differ in field order, not only in + // width, so the offsets cannot be derived from the word size. + int headerSize = is64 ? 64 : 52; + int phoffOff = is64 ? 0x20 : 0x1C; + int phentsizeOff = is64 ? 0x36 : 0x2A; + int phnumOff = is64 ? 0x38 : 0x2C; + int minPhentsize = is64 ? 56 : 32; + int pOffsetOff = is64 ? 0x08 : 0x04; + int pFileszOff = is64 ? 0x20 : 0x10; + int pAlignOff = is64 ? 0x30 : 0x1C; + + assertTrue(elf.length >= headerSize, + name + " is truncated: " + elf.length + " bytes, shorter than " + + "the " + headerSize + "-byte ELF header"); + ByteBuffer buffer = ByteBuffer.wrap(elf); + buffer.order((elf[5] & 0xff) == 1 ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN); + long phoff = is64 ? buffer.getLong(phoffOff) + : buffer.getInt(phoffOff) & 0xffffffffL; + int phentsize = buffer.getShort(phentsizeOff) & 0xffff; + int phnum = buffer.getShort(phnumOff) & 0xffff; + assertTrue(phentsize >= minPhentsize && phnum > 0 && phoff > 0 + && phoff + (long) phnum * phentsize <= elf.length, + name + " has a malformed ELF program header table"); + + List alignments = new ArrayList(); + for (int i = 0; i < phnum; i++) { + int offset = (int) (phoff + (long) i * phentsize); + if (buffer.getInt(offset) != PT_LOAD) { + continue; + } + long pOffset = is64 ? buffer.getLong(offset + pOffsetOff) + : buffer.getInt(offset + pOffsetOff) & 0xffffffffL; + long pFilesz = is64 ? buffer.getLong(offset + pFileszOff) + : buffer.getInt(offset + pFileszOff) & 0xffffffffL; + // Program headers sit near the front of the file, so they parse + // even when the segments they describe are not all there. + assertTrue(pOffset + pFilesz <= elf.length, + name + " is truncated: a PT_LOAD segment ends at " + + (pOffset + pFilesz) + ", past the " + elf.length + + "-byte file"); + alignments.add(Long.valueOf(is64 ? buffer.getLong(offset + pAlignOff) + : buffer.getInt(offset + pAlignOff) & 0xffffffffL)); + } + assertTrue(!alignments.isEmpty(), name + " has no PT_LOAD segments"); + if (!is64) { + return null; + } + long[] result = new long[alignments.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = alignments.get(i).longValue(); + } + return result; + } + + private static byte[] readFully(ZipFile zip, ZipEntry entry) throws Exception { + InputStream input = zip.getInputStream(entry); + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] chunk = new byte[8192]; + int read; + while ((read = input.read(chunk)) > 0) { + out.write(chunk, 0, read); + } + return out.toByteArray(); + } finally { + input.close(); + } + } + private static String read(String path) throws Exception { File file = new File(path); assertTrue(file.isFile(), "Missing " + file.getAbsolutePath()); diff --git a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java index 9e1820cee97..ac951abbc16 100644 --- a/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java +++ b/maven/platform-feature-catalog/src/main/java/com/codename1/build/shared/PlatformFeatureCatalog.java @@ -394,8 +394,16 @@ public final class PlatformFeatureCatalog { .iosDependenciesUnsupportedOnArm64Simulator() .description("ML Kit iOS Smart Reply backend")); + // The Android AAR declares minSdk 21, which is what its native code has + // always required -- the NDK's own floor is API 21, so the slices were + // compiled against 21 even while the AAR manifest still said 19. + // Without this floor the manifest merger rejects a default Codename One + // project, whose android.min_sdk_version is 19, with + // "uses-sdk:minSdkVersion 19 cannot be smaller than version 21 declared + // in library". e.add(new Entry("com/codename1/ai/whisper/") .iosFrameworks("Accelerate") + .androidMinimumSdk(21) .description("On-device Whisper transcription (libwhisper.a ships with the cn1lib)")); // Low-level cross-platform camera API: live preview + frame diff --git a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java index 2c788e69d5a..806ac91d664 100644 --- a/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java +++ b/maven/platform-feature-catalog/src/test/java/com/codename1/build/shared/PlatformFeatureCatalogTest.java @@ -380,6 +380,19 @@ void stableDiffusionFlagsBigUpload() { "ONNX Runtime Android requires API 21"); } + @Test + void whisperRaisesTheAndroidFloorToTwentyOne() { + // The Android AAR declares minSdk 21, which its native code always + // required (the NDK floor is API 21). Without this floor the manifest + // merger rejects a default project (android.min_sdk_version 19) with + // "uses-sdk:minSdkVersion 19 cannot be smaller than version 21 + // declared in library". + PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); + acc.consume("com/codename1/ai/whisper/WhisperRecognizer"); + assertEquals(21, acc.minimumAndroidSdk(), + "cn1-ai-whisper-android.aar declares minSdk 21"); + } + @Test void builtInVisionDoesNotFlagBigUpload() { PlatformFeatureCatalog.Accumulator acc = new PlatformFeatureCatalog.Accumulator(); diff --git a/scripts/check-16k-page-alignment.py b/scripts/check-16k-page-alignment.py new file mode 100755 index 00000000000..b30abd922a3 --- /dev/null +++ b/scripts/check-16k-page-alignment.py @@ -0,0 +1,591 @@ +#!/usr/bin/env python3 +"""Reject 64-bit native libraries that are not 16 KB page aligned. + +Android 15 introduced devices with a 16 KB memory page size, and Google Play +requires every 64-bit shared library in an upload to be laid out for them: +new apps and updates targeting API 35+ have needed it since 2025-11-01, and +Wear OS uploads containing native code need it (alongside 64-bit slices) from +2026-09-15. See https://developer.android.com/guide/practices/page-sizes. + +"Laid out for them" means each `PT_LOAD` program header carries `p_align` of +at least 0x4000. Alignment is baked in at *link* time, so no packaging step +can repair it after the fact -- AGP aligns the ZIP entries, and bundletool +reports the result, but neither one can move a segment inside a prebuilt +`.so`. A library linked with a 4 KB assumption stays broken until it is +rebuilt, which is why this is a gate on the checked-in artifact rather than a +check on the generated app. + +The failure it prevents is quiet in exactly the way that matters: the app +builds, uploads, and runs on every 4 KB device the developer owns, and only +Play review or a 16 KB device says otherwise. And because the artifact is +Codename One's rather than the developer's, they cannot fix it themselves. + +The alignment rule is scoped twice. It covers 64-bit ELF only, because 16 KB +pages are a 64-bit feature and `armeabi-v7a`/`x86` never run on such a +device. And it covers only libraries Android packaging actually ships -- a +desktop Linux `.so` is built for 4 KB pages and is correct at 0x1000, and a +cn1lib can carry one (Cn1libMojo packages `linux`/`javase` resources into +`nativelinux`/`nativese`). Everything skipped is still reported under +--verbose, and the integrity checks below are NOT scoped: a `.so` that is not +an ELF, or one that is truncated, is a bug on any platform. + +Usage: + scripts/check-16k-page-alignment.py # every tracked artifact + scripts/check-16k-page-alignment.py PATH ... # only these files + scripts/check-16k-page-alignment.py --verbose # list everything scanned +""" + +import io +import os +import re +import struct +import subprocess +import sys +import zipfile + +# Google's required minimum `p_align` for a PT_LOAD segment. A library built +# by NDK r28+ gets this by default; r26/r27 need both +# -Wl,-z,max-page-size=16384 and -Wl,-z,common-page-size=16384. +REQUIRED_ALIGNMENT = 0x4000 + +# Containers and libraries are recognised by content, never by file +# extension. An extension list is a curated allow-list, and this one was +# already wrong: it knew `.aar`/`.apk`/`.aab`/`.jar`/`.zip` and so walked +# straight past every `.cn1lib` in the tree -- Codename One's own library +# format, a ZIP, and the standard packaging path by which a cn1lib ships +# Android natives. `tests/core/lib/QRScanner.cn1lib` reaches real `.so` +# files through `nativeand.zip` and then `ZBarScannerLibrary.aar`, and none +# of them were being inspected. +# +# Sniffing the magic bytes ends that class of bug instead of moving the +# list forward one entry: any container, whatever it is called, and any +# ELF, whatever it is called, is found. +ZIP_MAGIC = (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08") + +# Content decides what gets *scanned*; these names decide what gets +# *challenged*. They are not the old allow-list returning -- nothing is found +# by being on them, and a container with a name nobody anticipated is still +# picked up by its magic. They encode a second rule: a file whose name +# declares a format must actually be that format. Without it, a zeroed or +# truncated `libfoo.so` matches no magic, is classified as an ordinary file, +# and is skipped in silence -- the exact "could not look reads as clean" hole +# this gate exists to avoid, reachable by corrupting a library rather than +# misaligning it. +ARCHIVE_NAME_SUFFIXES = (".aar", ".apk", ".aab", ".jar", ".zip", ".cn1lib") + +# 16 KB pages are an Android/Google Play requirement, not a property of ELF. +# An ordinary desktop Linux x86_64 `.so` is built for 4 KB pages and is +# perfectly correct at 0x1000, so the alignment rule must apply only where +# Android packaging says the library ships to a device. Codename One really +# can carry such a file: Cn1libMojo.buildLinux() packages a cn1lib's +# `linux/src/main/resources` into `nativelinux`, and buildJavase() does the +# same for `nativese`. Without this scoping the workflow -- which reads every +# tracked file -- would block every PR over a valid artifact. +# +# Only the alignment rule is scoped. The integrity checks (a `.so` that is +# not an ELF, a truncated one) stay universal, because those are bugs on any +# platform. +ANDROID_CONTAINER_SUFFIXES = (".aar", ".apk", ".aab") + +# The Android payload a cn1lib carries, written by Cn1libMojo.buildAndroid(). +ANDROID_CN1LIB_PAYLOAD = "nativeand.zip" + +# Directory names Android packaging uses for native libraries, and the ABI +# directory that must sit directly inside one. Together they are what marks a +# path as Android even outside an AAR -- a loose `jniLibs/arm64-v8a/libfoo.so` +# in a port is still an Android artifact. +ANDROID_LIB_DIRS = frozenset(["jni", "lib", "jnilibs"]) +ANDROID_ABIS = frozenset([ + "armeabi", "armeabi-v7a", "arm64-v8a", "x86", "x86_64", + "mips", "mips64", "riscv64", +]) + +# An archive inside an archive is legitimate and, as the QRScanner cn1lib +# shows, three deep happens in practice. Bounded so a malformed or hostile +# ZIP cannot spin this gate forever, but with headroom: exceeding the limit +# is reported as a failure, so a limit set too close to real usage would +# raise false alarms rather than silently skip. +MAX_ARCHIVE_DEPTH = 8 + +# A whole-tree run that inspected nothing must not report success: a wrong +# working directory, or a `git ls-files` that came back empty, would +# otherwise read exactly like a clean tree. This repository ships one native +# artifact (the cn1-ai-whisper AAR) holding six 64-bit libraries. If that +# artifact is ever legitimately removed, lower this floor in the same commit +# rather than deleting the check. +MIN_LIBRARIES_SCANNED = 1 + +ELF_MAGIC = b"\x7fELF" +# Longest magic this gate sniffs, so callers know how many bytes to read. +MAGIC_LENGTH = 4 +ELFCLASS32 = 1 +ELFCLASS64 = 2 +ELFDATA2LSB = 1 +ELFDATA2MSB = 2 +PT_LOAD = 1 + +# Per ELF class: header size, the offsets of e_phoff / e_phentsize / e_phnum, +# the struct code for an address-sized word, the smallest legal program header +# entry, and the offsets of p_offset / p_filesz / p_align inside one. The two +# program header layouts differ in field order, not just in width, so the +# offsets cannot be derived from the word size. +ELF_LAYOUT = { + ELFCLASS64: (64, 0x20, 0x36, 0x38, "Q", 56, 0x08, 0x10, 0x20, 0x30), + ELFCLASS32: (52, 0x1C, 0x2A, 0x2C, "I", 32, 0x04, 0x08, 0x10, 0x1C), +} + +# e_type sits at 0x10 and e_machine at 0x12, in both classes. +E_TYPE_OFFSET = 0x10 +E_MACHINE_OFFSET = 0x12 +ET_DYN = 3 +ET_NAMES = {0: "ET_NONE", 1: "ET_REL", 2: "ET_EXEC", 3: "ET_DYN", 4: "ET_CORE"} +EM_ARM = 40 +EM_AARCH64 = 183 +EM_386 = 3 +EM_X86_64 = 62 +EM_MIPS = 8 +EM_RISCV = 243 + +# What an Android ABI directory asserts about the library inside it. The gate +# picks which rule to apply from that directory name, so when the file +# contradicts it the evidence is self-contradictory and NEITHER verdict is +# sound: exempting it as 32-bit and accepting it as an aligned 64-bit slice +# are both guesses. Reported rather than guessed, for the same reason an +# unreadable file is. Verified against every library in the tree before being +# switched on -- all 18 agree with their directory. +ANDROID_ABI_ELF = { + "armeabi": (ELFCLASS32, EM_ARM), + "armeabi-v7a": (ELFCLASS32, EM_ARM), + "arm64-v8a": (ELFCLASS64, EM_AARCH64), + "x86": (ELFCLASS32, EM_386), + "x86_64": (ELFCLASS64, EM_X86_64), + "mips": (ELFCLASS32, EM_MIPS), + "mips64": (ELFCLASS64, EM_MIPS), + "riscv64": (ELFCLASS64, EM_RISCV), +} + + +class MalformedElf(Exception): + """A `.so` payload that cannot be read as a well-formed ELF.""" + + +def read_elf_header(data): + """Return `(ei_class, e_type, e_machine, layout, prefix)`. + + Only the ELF header, so the caller can find out WHAT the file is before + committing to reading it as a shared library. A relocatable object from + `cc -c` legitimately has no program header table at all, and demanding + one of every ELF in the tree rejected it with "program header entry size + 0 is too small" -- a valid build artifact blocking the run. + + Raises MalformedElf for anything that is not a readable ELF header. + """ + if len(data) < 5 or data[:4] != ELF_MAGIC: + raise MalformedElf("not an ELF file") + + ei_class = data[4] + if ei_class not in ELF_LAYOUT: + raise MalformedElf("unknown ELF class 0x%02x" % ei_class) + layout = ELF_LAYOUT[ei_class] + header_size = layout[0] + + if len(data) < header_size: + raise MalformedElf( + "truncated: %d bytes, shorter than the %d-byte ELF header" + % (len(data), header_size)) + + endian = data[5] + if endian == ELFDATA2LSB: + prefix = "<" + elif endian == ELFDATA2MSB: + prefix = ">" + else: + raise MalformedElf("unknown ELF data encoding 0x%02x" % endian) + + e_type = struct.unpack_from(prefix + "H", data, E_TYPE_OFFSET)[0] + e_machine = struct.unpack_from(prefix + "H", data, E_MACHINE_OFFSET)[0] + return ei_class, e_type, e_machine, layout, prefix + + +def read_load_alignments(data, layout, prefix): + """Return the `p_align` of every PT_LOAD segment of a shared object. + + Raises MalformedElf for anything that does not parse -- a truncated + library, a corrupted one, or a file that merely happens to be named + `.so`. Nothing is skipped: an entry this function could not read is an + entry whose alignment nobody checked, and that must not come out reading + the same as one checked and found compliant. Returning "exempt" for an + unreadable payload would also make the exemption reachable by corruption + -- a truncated 64-bit library starts with the ELF magic and would be + waved through as if it were a 32-bit slice. + + Only called once the header says this really is a shared object, so the + program header table is genuinely required here. + """ + (header_size, phoff_off, phentsize_off, phnum_off, word, min_phentsize, + poffset_off, pvaddr_off, pfilesz_off, palign_off) = layout + + e_phoff = struct.unpack_from(prefix + word, data, phoff_off)[0] + e_phentsize = struct.unpack_from(prefix + "H", data, phentsize_off)[0] + e_phnum = struct.unpack_from(prefix + "H", data, phnum_off)[0] + + if e_phentsize < min_phentsize: + raise MalformedElf( + "program header entry size %d is too small" % e_phentsize) + if e_phoff == 0 or e_phnum == 0: + raise MalformedElf("no program headers") + if e_phoff + e_phnum * e_phentsize > len(data): + raise MalformedElf("program header table runs past end of file") + + alignments = [] + for index in range(e_phnum): + offset = e_phoff + index * e_phentsize + # p_type is the first word of a program header in both classes. + p_type = struct.unpack_from(prefix + "I", data, offset)[0] + if p_type != PT_LOAD: + continue + # A segment that claims to run past the end of the file is a + # truncated artifact. Its program headers can still parse -- they sit + # near the front -- so without this the alignment reads as correct on + # a library whose contents are simply not all there. + p_offset = struct.unpack_from(prefix + word, data, offset + poffset_off)[0] + p_filesz = struct.unpack_from(prefix + word, data, offset + pfilesz_off)[0] + if p_offset + p_filesz > len(data): + raise MalformedElf( + "truncated: PT_LOAD segment ends at %d, past the %d-byte file" + % (p_offset + p_filesz, len(data))) + alignment = struct.unpack_from( + prefix + word, data, offset + palign_off)[0] + # ELF permits 0 and 1 ("no alignment required") or a power of two, + # and nothing else. Without this a malformed 0x4001 clears a + # minimum-only comparison and is announced as 16 KB compliant, which + # is the gate accepting a value no loader would honour. Reported as + # malformed rather than misaligned, because it is not a number a + # linker produces and "rebuild it aligned" is the wrong advice. + if alignment & (alignment - 1): + raise MalformedElf( + "PT_LOAD p_align 0x%x is not a power of two" % alignment) + # ELF requires p_vaddr and p_offset to be congruent modulo p_align; + # a segment that is not cannot be mapped as it declares itself, so + # calling it aligned would vouch for a library no loader can honour. + p_vaddr = struct.unpack_from(prefix + word, data, offset + pvaddr_off)[0] + if alignment > 1 and (p_vaddr % alignment) != (p_offset % alignment): + raise MalformedElf( + "PT_LOAD p_vaddr 0x%x and p_offset 0x%x are not congruent " + "modulo p_align 0x%x" % (p_vaddr, p_offset, alignment)) + alignments.append(alignment) + if not alignments: + raise MalformedElf("no PT_LOAD segments") + return alignments + + +def is_android_payload(label): + """True when the accumulated label shows Android packaging. + + `label` is the scan path, with `!` separating each archive from what was + read out of it, so this sees the whole chain -- an `.so` reached through + `QRScanner.cn1lib!nativeand.zip!ZBarScannerLibrary.aar!jni/...` is Android + because of containers three levels up. + """ + for part in label.split("!"): + base = os.path.basename(part).lower() + if base.endswith(ANDROID_CONTAINER_SUFFIXES): + return True + if base == ANDROID_CN1LIB_PAYLOAD: + return True + segments = [segment.lower() for segment in part.replace("\\", "/").split("/")] + for index in range(len(segments) - 1): + if (segments[index] in ANDROID_LIB_DIRS + and segments[index + 1] in ANDROID_ABIS): + return True + return False + + +# A shared-library soname version is numeric: `libfoo.so.1`, `libfoo.so.1.2`. +SONAME_VERSION = re.compile(r"[0-9]+(\.[0-9]+)*\Z") + + +def android_abi_for(label): + """The Android ABI directory governing this library, or None. + + The innermost match wins: the directory closest to the file is the one + whose contents Android will load for that ABI. + """ + found = None + for part in label.split("!"): + segments = [segment.lower() + for segment in part.replace("\\", "/").split("/")] + for index in range(len(segments) - 1): + if (segments[index] in ANDROID_LIB_DIRS + and segments[index + 1] in ANDROID_ABIS): + found = segments[index + 1] + return found + + +def looks_like_shared_library(name): + """True for `libfoo.so` and for a versioned `libfoo.so.1` or `.so.1.2`. + + The version has to be numeric. A looser "contains `.so.`" test also + swallows the metadata that sits beside a native library -- a + `libfoo.so.sha256` checksum, a `libfoo.so.asc` signature -- and this gate + reads every tracked path, so it would fail the whole run with "not an ELF + file" over a text file. Being strict here costs nothing: a file that is + genuinely a library is still found by its magic whatever it is called. + """ + base = os.path.basename(name).lower() + if base.endswith(".so"): + return True + _, separator, version = base.partition(".so.") + return bool(separator) and SONAME_VERSION.match(version) is not None + + +def looks_like_archive(name): + return os.path.basename(name).lower().endswith(ARCHIVE_NAME_SUFFIXES) + + +def classify(name, head): + """Return "elf", "zip" or None for one file, from its name and content. + + Declarations are answered before discoveries, and for both formats + symmetrically. A name that declares a format routes to that format's + reader whatever the bytes turn out to be, so a file that is not what it + says it is gets reported instead of quietly reinterpreted. Checking the + library name first but leaving the archive name as a fallback after + content was an asymmetry with real consequences: a valid ELF left in + place of an `.aar` was read as a library and counted as a compliant + Android slice, and the same bytes named `.cn1lib` were waved through as + non-Android -- either way a destroyed container kept the run green. + + Only once a name declares nothing does content decide, which is what + finds a container or a library under a name nobody anticipated. + + A deliberately corrupt archive fixture would now be reported. None + exists, and renaming it would be the fix; the alternative is a gate that + cannot tell a broken container from an absent one. + """ + if looks_like_shared_library(name): + return "elf" + if looks_like_archive(name): + return "zip" + if head[:4] == ELF_MAGIC: + return "elf" + if head[:4] in ZIP_MAGIC: + return "zip" + return None + + +class Scanner(object): + def __init__(self, verbose): + self.verbose = verbose + self.failures = [] + self.libraries_scanned = 0 + self.skipped_32bit = 0 + self.skipped_non_android = 0 + self.skipped_not_a_library = 0 + self.has_alignment_failure = False + + def check_library(self, label, data): + try: + ei_class, e_type, e_machine, layout, prefix = read_elf_header(data) + except MalformedElf as error: + self.failures.append("%s: %s" % (label, error)) + return + + # Only a shared object is a shared library. Whether a file that is + # not one is a finding or none of this gate's business depends on + # what it claimed to be: a name saying `.so`, or a place in an + # Android ABI directory, is a claim, and a relocatable object or an + # executable there cannot be loaded. An ELF found purely by its magic + # under a name that claims nothing -- a checked-in `cc -c` object, + # say -- claimed nothing and is simply not what this gate inspects. + declared = (looks_like_shared_library(label.split("!")[-1]) + or android_abi_for(label) is not None) + if e_type != ET_DYN: + if declared: + self.failures.append( + "%s: not a shared object (%s)" + % (label, ET_NAMES.get(e_type, "e_type %d" % e_type))) + else: + self.skipped_not_a_library += 1 + if self.verbose: + print(" not a shared object (%s), not inspected: %s" + % (ET_NAMES.get(e_type, e_type), label)) + return + + try: + alignments = read_load_alignments(data, layout, prefix) + except MalformedElf as error: + self.failures.append("%s: %s" % (label, error)) + return + + # The ABI directory is the evidence this gate uses to decide which + # rule applies, so a file that contradicts it leaves no sound verdict + # to reach -- exempting it as 32-bit and accepting it as an aligned + # 64-bit slice would both be guesses. + abi = android_abi_for(label) + expected = ANDROID_ABI_ELF.get(abi) + if expected is not None and (ei_class, e_machine) != expected: + self.failures.append( + "%s: packaged under %s but the file is %d-bit e_machine %d, " + "which that directory cannot hold" + % (label, abi, 64 if ei_class == ELFCLASS64 else 32, + e_machine)) + return + + if ei_class != ELFCLASS64: + self.skipped_32bit += 1 + if self.verbose: + print(" 32-bit (not subject to 16 KB pages): %s" % label) + return + if not is_android_payload(label): + # Parsed and sound, but not shipped to an Android device, so the + # Play requirement does not reach it. + self.skipped_non_android += 1 + if self.verbose: + print(" not Android packaging (16 KB rule does not apply): %s" + % label) + return + + self.libraries_scanned += 1 + worst = min(alignments) + if worst < REQUIRED_ALIGNMENT: + self.has_alignment_failure = True + self.failures.append( + "%s: PT_LOAD p_align 0x%x, needs 0x%x or more" + % (label, worst, REQUIRED_ALIGNMENT)) + elif self.verbose: + print(" ok 0x%x: %s" % (worst, label)) + + def scan_archive(self, label, data, depth): + if depth > MAX_ARCHIVE_DEPTH: + # Reported for the same reason an unreadable archive is: whatever + # is nested this deep went unexamined, and the run must not call + # itself clean over it. + self.failures.append( + "%s: nested more than %d archives deep, not inspected" + % (label, MAX_ARCHIVE_DEPTH)) + return + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + except (zipfile.BadZipFile, OSError) as error: + # Reported, never skipped. An archive this gate cannot open is one + # whose libraries it did not look at, and "could not look" must not + # come out reading the same as "looked and found nothing" -- a + # truncated artifact would then be waved through. Every archive in + # the tree opens today, so there is no benign case to tolerate. + self.failures.append("%s: cannot read archive (%s)" % (label, error)) + return + with archive: + for entry in archive.infolist(): + if entry.is_dir(): + continue + # Only the magic is decompressed for entries that turn out to + # be neither, which is nearly all of them. + try: + with archive.open(entry) as stream: + head = stream.read(MAGIC_LENGTH) + except (zipfile.BadZipFile, OSError, RuntimeError) as error: + self.failures.append( + "%s!%s: cannot read entry (%s)" + % (label, entry.filename, error)) + continue + kind = classify(entry.filename, head) + if kind is None: + continue + nested = "%s!%s" % (label, entry.filename) + if kind == "elf": + self.check_library(nested, archive.read(entry)) + else: + self.scan_archive(nested, archive.read(entry), depth + 1) + + def scan_path(self, path): + with open(path, "rb") as handle: + head = handle.read(MAGIC_LENGTH) + kind = classify(path, head) + if kind is None: + return + with open(path, "rb") as handle: + data = handle.read() + if kind == "elf": + self.check_library(path, data) + else: + self.scan_archive(path, data, 1) + + +def tracked_files(): + """Every tracked path, as an absolute name. + + Resolved from the repository root rather than the working directory: run + from a subdirectory, `git ls-files` lists only what is under it, and the + whole-tree mode would quietly inspect a fraction of the tree while still + calling itself a whole-tree run. + """ + root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], + universal_newlines=True).strip() + output = subprocess.check_output( + ["git", "-C", root, "ls-files", "-z"], universal_newlines=False) + return [os.path.join(root, name.decode("utf-8")) + for name in output.split(b"\0") if name] + + +def main(argv): + verbose = "--verbose" in argv or "-v" in argv + paths = [arg for arg in argv if not arg.startswith("-")] + whole_tree = not paths + if whole_tree: + # Every tracked file. scan_path sniffs and ignores what is neither a + # container nor an ELF, so nothing has to be guessed from a name. + paths = tracked_files() + + scanner = Scanner(verbose) + for path in paths: + if not os.path.isfile(path): + if whole_tree: + # Tracked but deleted in the working tree. Nothing to read, + # and nothing wrong with the tree as committed. + continue + print("Missing %s" % path, file=sys.stderr) + return 2 + scanner.scan_path(path) + + if whole_tree and scanner.libraries_scanned < MIN_LIBRARIES_SCANNED: + print("check-16k-page-alignment: found %d 64-bit native libraries, " + "expected at least %d -- the scan looked at nothing, which is " + "not the same as a clean tree." + % (scanner.libraries_scanned, MIN_LIBRARIES_SCANNED), + file=sys.stderr) + return 2 + + if scanner.failures: + print("16 KB page alignment check failed:", file=sys.stderr) + for failure in scanner.failures: + print(" %s" % failure, file=sys.stderr) + # Only offered when something was actually misaligned. A truncated or + # unreadable artifact is a different problem, and "rebuild with a + # newer NDK" would send the reader off in the wrong direction. + if scanner.has_alignment_failure: + print("", file=sys.stderr) + print("Rebuild the misaligned libraries with NDK r28 or newer, " + "which emits 16 KB aligned libraries by default. On NDK " + "r26/r27 pass both -Wl,-z,max-page-size=16384 and " + "-Wl,-z,common-page-size=16384, and note that the NDK's own " + "prebuilt libc++_shared.so and libomp.so are 4 KB aligned " + "there and cannot be fixed by a linker flag.", + file=sys.stderr) + print("See " + "https://developer.android.com/guide/practices/page-sizes", + file=sys.stderr) + return 1 + + print("check-16k-page-alignment: %d Android 64-bit native libraries " + "aligned to 0x%x or more (%d 32-bit and %d non-Android libraries " + "are not subject to the rule; %d ELF files are not shared objects)" + % (scanner.libraries_scanned, REQUIRED_ALIGNMENT, + scanner.skipped_32bit, scanner.skipped_non_android, + scanner.skipped_not_a_library)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/test_check_16k_page_alignment.py b/scripts/test_check_16k_page_alignment.py new file mode 100644 index 00000000000..8f47c5e3ac4 --- /dev/null +++ b/scripts/test_check_16k_page_alignment.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +"""Pins the classification boundaries of check-16k-page-alignment.py. + +Every case here is one a review caught on PR #5688, and each was a variant of +the same mistake: something the gate could not read, or was not asked about, +came out looking identical to something it had read and found compliant. The +opposite failure showed up too -- a valid artifact rejected because the rule +was applied where it does not apply. + +So the three verdicts are pinned deliberately and separately: + + enforced a 64-bit library under Android packaging + exempt 32-bit, or not Android packaging -- reported, never failed + reported unreadable, truncated, or not an ELF at all -- always a failure + +ELF payloads are synthesised rather than copied from a build, so the test is +self-contained and does not need a toolchain or a checked-in fixture. +""" + +import io +import os +import struct +import types +import unittest +import zipfile + +GATE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "check-16k-page-alignment.py") + + +def load_gate(): + """Load the gate from source, never from `__pycache__`. + + importlib would happily serve a stale `.pyc` -- `python3 -m py_compile` + on the gate leaves one behind, and it really did shadow an edit here, + reporting a behaviour the source no longer had. A pinning test that can + pass against code that no longer exists is worse than no test, so the + source is compiled every run. + """ + with io.open(GATE_PATH, encoding="utf-8") as handle: + source = handle.read() + module = types.ModuleType("check_16k_page_alignment") + module.__file__ = GATE_PATH + exec(compile(source, GATE_PATH, "exec"), module.__dict__) + return module + + +gate = load_gate() + +ALIGNED_DEFAULT = 0x4000 + +ELF64_HEADER = 64 +ELF64_PHENTSIZE = 56 +ELF32_HEADER = 52 +ELF32_PHENTSIZE = 32 + + +def elf64(p_align, machine=0xB7, e_type=3): + """A minimal little-endian 64-bit ELF with one PT_LOAD segment.""" + total = ELF64_HEADER + ELF64_PHENTSIZE + header = struct.pack( + "<4sBBBB8xHHIQQQIHHHHHH", + b"\x7fELF", 2, 1, 1, 0, # magic, ELFCLASS64, LSB, version + e_type, machine, 1, # e_type, machine, version + 0, ELF64_HEADER, 0, 0, # entry, phoff, shoff, flags + ELF64_HEADER, ELF64_PHENTSIZE, 1, 0, 0, 0) + phdr = struct.pack( + " nativeand.zip -> aar -> .so, the depth the tree really + # uses, reached through a container the extension list did not know. + inner = zipped({"jni/arm64-v8a/libfoo.so": elf64(MISALIGNED)}) + payload = zipped({"nativeand.zip": zipped({"inner.aar": inner})}) + failures, _ = scan("QRScanner.cn1lib", payload) + self.assertEqual(1, len(failures), failures) + self.assertIn("p_align 0x1000", failures[0]) + + +class ExemptionsAreNotFailures(unittest.TestCase): + def test_32_bit_library_is_exempt(self): + failures, scanner = scan("app.aar!jni/armeabi-v7a/libfoo.so", + elf32(MISALIGNED)) + self.assertEqual([], failures) + self.assertEqual(1, scanner.skipped_32bit) + self.assertEqual(0, scanner.libraries_scanned) + + def test_desktop_64_bit_library_is_exempt_at_4kb(self): + # A desktop Linux .so is correct at 0x1000; failing it would block + # every PR over a valid artifact. + failures, scanner = scan("L.cn1lib!nativelinux.zip!libfoo.so", + elf64(MISALIGNED)) + self.assertEqual([], failures) + self.assertEqual(1, scanner.skipped_non_android) + self.assertEqual(0, scanner.libraries_scanned) + + def test_same_bytes_enforced_or_exempt_by_packaging_alone(self): + payload = elf64(MISALIGNED) + android, _ = scan(ANDROID_64, payload) + desktop, _ = scan("L.cn1lib!nativelinux.zip!libfoo.so", payload) + self.assertEqual(1, len(android)) + self.assertEqual([], desktop) + + +class UnreadableIsAlwaysReported(unittest.TestCase): + """The core rule: "could not look" must never read as "looked, clean".""" + + def test_so_that_is_not_an_elf_fails(self): + for payload in [b"\x00" * 4096, b"this is not an elf"]: + failures, _ = scan(ANDROID_64, payload) + self.assertEqual(1, len(failures), failures) + self.assertIn("not an ELF file", failures[0]) + + def test_corrupt_library_is_not_mistaken_for_32_bit(self): + # A truncated 64-bit ELF still begins with the ELF magic, so the + # exemption for an ABI the rule does not cover was reachable by + # corrupting a library. + failures, scanner = scan(ANDROID_64, elf64(ALIGNED)[:40]) + self.assertEqual(1, len(failures), failures) + self.assertIn("truncated", failures[0]) + self.assertEqual(0, scanner.skipped_32bit) + + def test_segment_running_past_end_of_file_fails(self): + # Program headers sit near the front, so a library truncated mid-file + # parses and reports correct alignment for contents that are absent. + payload = bytearray(elf64(ALIGNED)) + struct.pack_into("