diff --git a/docs/contributing-code/source_code.md b/docs/contributing-code/source_code.md index ddb12f88a24..ddf35394fb7 100644 --- a/docs/contributing-code/source_code.md +++ b/docs/contributing-code/source_code.md @@ -38,3 +38,12 @@ permalink: /contributing-code/source-code/ * **platform_requirements.txt** - platform dependent package list. * **bower.json** - component dependencies for Polymer 2. * **butler.py** - helper script for various command line tasks (e.g. testing, deployment). + +## Pitfalls + +* **App Engine imports** - Directories listed in + [`.gcloudignore`](https://github.com/google/clusterfuzz/blob/master/src/appengine/.gcloudignore) + are not uploaded to App Engine. Shared modules cannot use top-level imports + from these directories, as this causes a `ModuleNotFoundError` on App Engine + startup. Instead, import them locally inside the functions or methods where + they are used. diff --git a/src/clusterfuzz/_internal/build_management/build_archive.py b/src/clusterfuzz/_internal/build_management/build_archive.py index 7fa22f6486d..6e04f6e1d9e 100644 --- a/src/clusterfuzz/_internal/build_management/build_archive.py +++ b/src/clusterfuzz/_internal/build_management/build_archive.py @@ -96,7 +96,9 @@ def list_fuzz_targets(self) -> List[str]: The list of fuzz targets. """ if self._fuzz_targets is None: - # Import here as this path is not available in App Engine context. + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils self._fuzz_targets = { @@ -207,7 +209,9 @@ def get_target_dependencies( @override def find_fuzz_targets(self) -> List[str]: - # Import here as this path is not available in App Engine context. + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils return [ @@ -328,7 +332,9 @@ def __init__(self, 'archive_schema_version field') self._archive_schema_version = default_archive_schema_version - # Import here as this path is not available in App Engine context. + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils self._manifest_fuzz_targets = ( diff --git a/src/clusterfuzz/_internal/build_management/build_manager.py b/src/clusterfuzz/_internal/build_management/build_manager.py index 102d5a4f68f..b2a094b73b4 100644 --- a/src/clusterfuzz/_internal/build_management/build_manager.py +++ b/src/clusterfuzz/_internal/build_management/build_manager.py @@ -330,7 +330,9 @@ def delete(self): def _read_schema_version_from_manifest(build_dir: str) -> int: """Reads archive_schema_version from clusterfuzz_manifest.json.""" - # Import here as this path is not available in App Engine context. + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils manifest = fuzzer_utils.read_chrome_manifest(build_dir) @@ -369,7 +371,9 @@ def _patch_rpaths(build_dir: str, app_path_env: str): return if environment.is_engine_fuzzer_job(): - # Import here as this path is not available in App Engine context. + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils for target_path in fuzzer_utils.get_fuzz_targets(build_dir): @@ -607,7 +611,9 @@ def _unpack_build(self, def _get_fuzz_targets_from_dir(self, build_dir): """Get iterator of fuzz targets from build dir.""" - # Import here as this path is not available in App Engine context. + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils for path in fuzzer_utils.get_fuzz_targets(build_dir): @@ -1357,6 +1363,9 @@ def setup_regular_build(revision, build_class = RegularBuild if environment.is_trusted_host(): + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.untrusted_runner import build_setup_host build_class = build_setup_host.RemoteRegularBuild elif environment.platform() == 'FUCHSIA': @@ -1380,7 +1389,9 @@ def setup_regular_build(revision, # Additional binaries to pull (for fuzzing engines such as Centipede). extra_bucket_path = get_bucket_path('EXTRA_BUILD_BUCKET_PATH') if extra_bucket_path and not build.is_discovery: - # Import here as this path is not available in App Engine context. + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils extra_build_urls = get_build_urls_list(extra_bucket_path) extra_build_url = revisions.find_build_url(extra_bucket_path, @@ -1426,6 +1437,9 @@ def setup_symbolized_builds(revision): build_class = SymbolizedBuild if environment.is_trusted_host(): + # `clusterfuzz._internal.bot` has to be imported locally since it is not + # uploaded to GCP with App Engine context. See: + # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls from clusterfuzz._internal.bot.untrusted_runner import build_setup_host build_class = build_setup_host.RemoteSymbolizedBuild # pylint: disable=no-member