Reorganize and clarify API doc generator (#132353)

## Description This cleans up a lot of issues with the API doc generation. Here are the main changes: - Rename `dartdoc.dart` to `create_api_docs.dart` - Move the bulk of the operations out of `dev/bots/docs.sh` into `create_api_docs.dart`. - Delete `dashing_postprocess.dart` and `java_and_objc.dart` and incorporate those operations into `create_api_docs.dart`. - Refactor the doc generation into more understandable classes - Bump the snippets tool version to 0.4.0 (the latest one) - Centralize the information gathering about the Flutter repo into the new `FlutterInformation` class. - Clean up the directory handling, and convert to using the `file` package for all file and directory paths. - Add an `--output` option to docs.sh that specifies the location of the output ZIP file containing the docs. - Defaults to placing the output in `dev/docs/api_docs.zip` (i.e. where the previous code generates the file). - Moved all document generation into a temporary folder that is removed once the documents are generated, to avoid VSCode and other IDEs trying to index the thousands of HTML and JS files in the docs output. - Updated pubspec dependencies. ## Tests - Added tests for doc generation.
2023-08-15 15:12:16 -07:00
parent 301577a34f
commit 899a29f830
8 changed files with 1384 additions and 938 deletions
--- a/dev/bots/docs.sh
+++ b/dev/bots/docs.sh
@@ -16,102 +16,13 @@ function script_location() {
  cd -P "$(dirname "$script_location")" >/dev/null && pwd
 }

-function generate_docs() {
-    # Install and activate dartdoc.
-    # When updating to a new dartdoc version, please also update
-    # `dartdoc_options.yaml` to include newly introduced error and warning types.
-    "$DART" pub global activate dartdoc 6.3.0
-
-    # Install and activate the snippets tool, which resides in the
-    # assets-for-api-docs repo:
-    # https://github.com/flutter/assets-for-api-docs/tree/master/packages/snippets
-    "$DART" pub global activate snippets 0.3.1
-
-    # This script generates a unified doc set, and creates
-    # a custom index.html, placing everything into dev/docs/doc.
-    (cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get)
-    (cd "$FLUTTER_ROOT/dev/tools" && "$DART" pub get)
-    (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev --enable-asserts "$FLUTTER_ROOT/dev/tools/dartdoc.dart")
-    (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev --enable-asserts "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart")
-}
-
-# Zip up the docs so people can download them for offline usage.
-function create_offline_zip() {
-  # Must be run from "$FLUTTER_ROOT/dev/docs"
-  echo "$(date): Zipping Flutter offline docs archive."
-  rm -rf flutter.docs.zip doc/offline
-  (cd ./doc; zip -r -9 -q ../flutter.docs.zip .)
-}
-
-# Generate the docset for Flutter docs for use with Dash, Zeal, and Velocity.
-function create_docset() {
-  # Must be run from "$FLUTTER_ROOT/dev/docs"
-  # Must have dashing installed: go get -u github.com/technosophos/dashing
-  # Dashing produces a LOT of log output (~30MB), so we redirect it, and just
-  # show the end of it if there was a problem.
-  echo "$(date): Building Flutter docset."
-  rm -rf flutter.docset
-  # If dashing gets stuck, Cirrus will time out the build after an hour, and we
-  # never get to see the logs. Thus, we run it in the background and tail the logs
-  # while we wait for it to complete.
-  dashing_log=/tmp/dashing.log
-  dashing build --source ./doc --config ./dashing.json > $dashing_log 2>&1 &
-  dashing_pid=$!
-  wait $dashing_pid && \
-  cp ./doc/flutter/static-assets/favicon.png ./flutter.docset/icon.png && \
-  "$DART" --disable-dart-dev --enable-asserts ./dashing_postprocess.dart && \
-  tar cf flutter.docset.tar.gz --use-compress-program="gzip --best" flutter.docset
-  if [[ $? -ne 0 ]]; then
-      >&2 echo "Dashing docset generation failed"
-      tail -200 $dashing_log
-      exit 1
-  fi
-}
-
-function deploy_docs() {
-    case "$LUCI_BRANCH" in
-        master)
-            echo "$(date): Updating $LUCI_BRANCH docs: https://master-api.flutter.dev/"
-            # Disable search indexing on the master staging site so searches get only
-            # the stable site.
-            echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
-            ;;
-        stable)
-            echo "$(date): Updating $LUCI_BRANCH docs: https://api.flutter.dev/"
-            # Enable search indexing on the master staging site so searches get only
-            # the stable site.
-            echo -e "# All robots welcome!" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
-            ;;
-        *)
-            >&2 echo "Docs deployment cannot be run on the $LUCI_BRANCH branch."
-            exit 0
-    esac
-}
-
-# Move the offline archives into place, after all the processing of the doc
-# directory is done. This avoids the tools recursively processing the archives
-# as part of their process.
-function move_offline_into_place() {
-  # Must be run from "$FLUTTER_ROOT/dev/docs"
-  echo "$(date): Moving offline data into place."
-  mkdir -p doc/offline
-  mv flutter.docs.zip doc/offline/flutter.docs.zip
-  du -sh doc/offline/flutter.docs.zip
-  if [[ "$LUCI_BRANCH" == "stable" ]]; then
-    echo -e "<entry>\n  <version>${FLUTTER_VERSION_STRING}</version>\n  <url>https://api.flutter.dev/offline/flutter.docset.tar.gz</url>\n</entry>" > doc/offline/flutter.xml
-  else
-    echo -e "<entry>\n  <version>${FLUTTER_VERSION_STRING}</version>\n  <url>https://master-api.flutter.dev/offline/flutter.docset.tar.gz</url>\n</entry>" > doc/offline/flutter.xml
-  fi
-  mv flutter.docset.tar.gz doc/offline/flutter.docset.tar.gz
-  du -sh doc/offline/flutter.docset.tar.gz
-}
-
 # So that users can run this script from anywhere and it will work as expected.
 SCRIPT_LOCATION="$(script_location)"
 # Sets the Flutter root to be "$(script_location)/../..": This script assumes
 # that it resides two directory levels down from the root, so if that changes,
 # then this line will need to as well.
 FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"
+export FLUTTER_ROOT

 echo "$(date): Running docs.sh"

@@ -124,31 +35,106 @@ FLUTTER_BIN="$FLUTTER_ROOT/bin"
 DART_BIN="$FLUTTER_ROOT/bin/cache/dart-sdk/bin"
 FLUTTER="$FLUTTER_BIN/flutter"
 DART="$DART_BIN/dart"
-export PATH="$FLUTTER_BIN:$DART_BIN:$PATH"
+PATH="$FLUTTER_BIN:$DART_BIN:$PATH"

-# Make sure dart is installed by invoking Flutter to download it.
-# This also creates the 'version' file.
-FLUTTER_VERSION=$("$FLUTTER" --version --machine)
+# Make sure dart is installed by invoking Flutter to download it if it is missing.
+# Also make sure the flutter command is ready to run before capturing output from
+# it: if it has to rebuild itself or something, it'll spoil our JSON output.
+"$FLUTTER" > /dev/null 2>&1
+FLUTTER_VERSION="$("$FLUTTER" --version --machine)"
 export FLUTTER_VERSION
-FLUTTER_VERSION_STRING=$(cat "$FLUTTER_ROOT/version")

 # If the pub cache directory exists in the root, then use that.
 FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache"
 if [[ -d "$FLUTTER_PUB_CACHE" ]]; then
  # This has to be exported, because pub interprets setting it to the empty
  # string in the same way as setting it to ".".
-  export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
+  PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
+  export PUB_CACHE
 fi

-generate_docs
-# Skip publishing docs for PRs and release candidate branches
-if [[ -n "$LUCI_CI" && -z "$LUCI_PR" ]]; then
-  (cd "$FLUTTER_ROOT/dev/docs"; create_offline_zip)
-  (cd "$FLUTTER_ROOT/dev/docs"; create_docset)
-  (cd "$FLUTTER_ROOT/dev/docs"; move_offline_into_place)
-  deploy_docs
-fi
+OUTPUT_DIR=$(mktemp -d /tmp/dartdoc.XXXXX)
+DOC_DIR="$OUTPUT_DIR/doc"

-# Zip docs
-cd "$FLUTTER_ROOT/dev/docs"
-zip -r api_docs.zip doc
+function usage() {
+  echo "Usage: $(basename "${BASH_SOURCE[0]}") [--keep-temp] [--output <output.zip>]"
+  echo ""
+  echo "  --keep-temp            Do not delete the temporary output directory created while generating docs."
+  echo "                         Normally the script deletes the temporary directory after generating the"
+  echo "                         output ZIP file."
+  echo "  --output <output.zip>  specifies where the output ZIP file containing the documentation data"
+  echo "                         will be written."
+  echo ""
+}
+
+function parse_args() {
+  local arg
+  local args=()
+  KEEP_TEMP=0
+  DESTINATION="$FLUTTER_ROOT/dev/docs/api_docs.zip"
+  while (( "$#" )); do
+    case "$1" in
+      --help)
+        usage
+        exit 0
+        ;;
+      --keep-temp)
+        KEEP_TEMP=1
+      ;;
+      --output)
+        DESTINATION="$2"
+        shift
+      ;;
+      *)
+        args=("${args[@]}" "$1")
+        ;;
+    esac
+    shift
+  done
+  if [[ ${#args[@]}  != 0 ]]; then
+    >&2 echo "ERROR: Unknown arguments: ${args[@]}"
+    usage
+    exit 1
+  fi
+}
+
+function generate_docs() {
+    # Install and activate dartdoc.
+    # When updating to a new dartdoc version, please also update
+    # `dartdoc_options.yaml` to include newly introduced error and warning types.
+    "$DART" pub global activate dartdoc 6.3.0
+
+    # Install and activate the snippets tool, which resides in the
+    # assets-for-api-docs repo:
+    # https://github.com/flutter/assets-for-api-docs/tree/master/packages/snippets
+    "$DART" pub global activate snippets 0.4.0
+
+    # This script generates a unified doc set, and creates
+    # a custom index.html, placing everything into DOC_DIR.
+
+    # Make sure that create_api_docs.dart has all the dependencies it needs.
+    (cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get)
+    (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev --enable-asserts "$FLUTTER_ROOT/dev/tools/create_api_docs.dart" --output-dir="$DOC_DIR")
+}
+
+function main() {
+  echo "Writing docs build temporary output to $DOC_DIR"
+  mkdir -p "$DOC_DIR"
+  generate_docs
+  # If the destination isn't an absolute path, make it into one.
+  if ! [[ "$DESTINATION" =~ ^/ ]]; then
+    DESTINATION="$PWD/$DESTINATION"
+  fi
+  # Zip up doc directory and write the output to the destination.
+  (cd "$OUTPUT_DIR"; zip -r -9 -q "$DESTINATION" ./doc)
+  if [[ $KEEP_TMP == 1 ]]; then
+    echo "Temporary document generation output left in $OUTPUT_DIR"
+  else
+    echo "Removing Temporary document generation output from $OUTPUT_DIR"
+    rm -rf "$OUTPUT_DIR"
+  fi
+  echo "Wrote docs ZIP file to $DESTINATION"
+}
+
+parse_args "$@"
+main