From 6ba8fa99872c9736be58c86903278faa8bfaf6c8 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Fri, 13 Sep 2019 12:49:03 -0700 Subject: [PATCH] Remove direct flutter tool usage of protobuf for encoding file caches (#40410) --- .../lib/src/build_system/file_hash_store.dart | 78 ++++++++++++--- .../lib/src/build_system/filecache.pb.dart | 96 ------------------- .../src/build_system/filecache.pbjson.dart | 25 ----- .../lib/src/build_system/filecache.proto | 18 ---- .../lib/src/commands/update_packages.dart | 1 - packages/flutter_tools/pubspec.yaml | 6 +- .../build_system/filecache_test.dart | 9 +- 7 files changed, 74 insertions(+), 159 deletions(-) delete mode 100644 packages/flutter_tools/lib/src/build_system/filecache.pb.dart delete mode 100644 packages/flutter_tools/lib/src/build_system/filecache.pbjson.dart delete mode 100644 packages/flutter_tools/lib/src/build_system/filecache.proto diff --git a/packages/flutter_tools/lib/src/build_system/file_hash_store.dart b/packages/flutter_tools/lib/src/build_system/file_hash_store.dart index 9abdef1e74..0df2a3e726 100644 --- a/packages/flutter_tools/lib/src/build_system/file_hash_store.dart +++ b/packages/flutter_tools/lib/src/build_system/file_hash_store.dart @@ -9,9 +9,58 @@ import 'dart:typed_data'; import 'package:crypto/crypto.dart'; import '../base/file_system.dart'; +import '../convert.dart'; import '../globals.dart'; import 'build_system.dart'; -import 'filecache.pb.dart' as pb; + +/// An encoded representation of all file hashes. +class FileStorage { + FileStorage(this.version, this.files); + + factory FileStorage.fromBuffer(Uint8List buffer) { + final Map json = jsonDecode(utf8.decode(buffer)); + final int version = json['version']; + final List rawCachedFiles = json['files']; + final List cachedFiles = [ + for (Map rawFile in rawCachedFiles) + FileHash.fromJson(rawFile) + ]; + return FileStorage(version, cachedFiles); + } + + final int version; + final List files; + + List toBuffer() { + final Map json = { + 'version': version, + 'files': [ + for (FileHash file in files) + file.toJson() + ], + }; + return utf8.encode(jsonEncode(json)); + } +} + +/// A stored file hash and path. +class FileHash { + FileHash(this.path, this.hash); + + factory FileHash.fromJson(Map json) { + return FileHash(json['path'], json['hash']); + } + + final String path; + final String hash; + + Object toJson() { + return { + 'path': path, + 'hash': hash, + }; + } +} /// A globally accessible cache of file hashes. /// @@ -42,7 +91,7 @@ class FileHashStore { static const String _kFileCache = '.filecache'; // The current version of the file cache storage format. - static const int _kVersion = 1; + static const int _kVersion = 2; /// Read file hashes from disk. void initialize() { @@ -51,12 +100,19 @@ class FileHashStore { return; } final List data = _cacheFile.readAsBytesSync(); - final pb.FileStorage fileStorage = pb.FileStorage.fromBuffer(data); + FileStorage fileStorage; + try { + fileStorage = FileStorage.fromBuffer(data); + } on FormatException { + printTrace('Filestorage format changed'); + _cacheFile.deleteSync(); + return; + } if (fileStorage.version != _kVersion) { _cacheFile.deleteSync(); return; } - for (pb.FileHash fileHash in fileStorage.files) { + for (FileHash fileHash in fileStorage.files) { previousHashes[fileHash.path] = fileHash.hash; } printTrace('Done initializing file store'); @@ -65,22 +121,22 @@ class FileHashStore { /// Persist file hashes to disk. void persist() { printTrace('Persisting file store'); - final pb.FileStorage fileStorage = pb.FileStorage(); - fileStorage.version = _kVersion; final File file = _cacheFile; if (!file.existsSync()) { file.createSync(); } + final List fileHashes = []; for (MapEntry entry in currentHashes.entries) { previousHashes[entry.key] = entry.value; } for (MapEntry entry in previousHashes.entries) { - final pb.FileHash fileHash = pb.FileHash(); - fileHash.path = entry.key; - fileHash.hash = entry.value; - fileStorage.files.add(fileHash); + fileHashes.add(FileHash(entry.key, entry.value)); } - final Uint8List buffer = fileStorage.writeToBuffer(); + final FileStorage fileStorage = FileStorage( + _kVersion, + fileHashes, + ); + final Uint8List buffer = fileStorage.toBuffer(); file.writeAsBytesSync(buffer); printTrace('Done persisting file store'); } diff --git a/packages/flutter_tools/lib/src/build_system/filecache.pb.dart b/packages/flutter_tools/lib/src/build_system/filecache.pb.dart deleted file mode 100644 index 84ca6528f8..0000000000 --- a/packages/flutter_tools/lib/src/build_system/filecache.pb.dart +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -/// -// Generated code. Do not modify. -// source: lib/src/build_system/filecache.proto -/// -// ignore_for_file: camel_case_types,non_constant_identifier_names,library_prefixes,unused_import,unused_shown_name, sort_constructors_first - -import 'dart:core' as $core show bool, Deprecated, double, int, List, Map, override, pragma, String, dynamic; - -import 'package:protobuf/protobuf.dart' as $pb; - -class FileHash extends $pb.GeneratedMessage { - factory FileHash() => create(); - - static final $pb.BuilderInfo _i = $pb.BuilderInfo('FileHash', package: const $pb.PackageName('flutter_tools')) - ..aOS(1, 'path') - ..aOS(2, 'hash') - ..hasRequiredFields = false; - - FileHash._() : super(); - - factory FileHash.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r); - - factory FileHash.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r); - - @$core.override - FileHash clone() => FileHash()..mergeFromMessage(this); - - @$core.override - FileHash copyWith(void Function(FileHash) updates) => super.copyWith(($core.dynamic message) => updates(message as FileHash)); - - @$core.override - $pb.BuilderInfo get info_ => _i; - - @$core.pragma('dart2js:noInline') - static FileHash create() => FileHash._(); - - // @$core.override - FileHash createEmptyInstance() => create(); // ignore: annotate_overrides - static $pb.PbList createRepeated() => $pb.PbList(); - static FileHash getDefault() => _defaultInstance ??= create()..freeze(); - static FileHash _defaultInstance; - - $core.String get path => $_getS(0, ''); - set path($core.String v) { $_setString(0, v); } - $core.bool hasPath() => $_has(0); - void clearPath() => clearField(1); - - $core.String get hash => $_getS(1, ''); - set hash($core.String v) { $_setString(1, v); } - $core.bool hasHash() => $_has(1); - void clearHash() => clearField(2); -} - -class FileStorage extends $pb.GeneratedMessage { - factory FileStorage() => create(); - static final $pb.BuilderInfo _i = $pb.BuilderInfo('FileHashStore', package: const $pb.PackageName('flutter_tools')) - ..a<$core.int>(1, 'version', $pb.PbFieldType.O3) - ..pc(2, 'files', $pb.PbFieldType.PM,FileHash.create) - ..hasRequiredFields = false; - - FileStorage._() : super(); - factory FileStorage.fromBuffer($core.List<$core.int> i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromBuffer(i, r); - factory FileStorage.fromJson($core.String i, [$pb.ExtensionRegistry r = $pb.ExtensionRegistry.EMPTY]) => create()..mergeFromJson(i, r); - - @$core.override - FileStorage clone() => FileStorage()..mergeFromMessage(this); - - @$core.override - FileStorage copyWith(void Function(FileStorage) updates) => super.copyWith(($core.dynamic message) => updates(message as FileStorage)); - - @$core.override - $pb.BuilderInfo get info_ => _i; - - @$core.pragma('dart2js:noInline') - static FileStorage create() => FileStorage._(); - - // @$core.override - FileStorage createEmptyInstance() => create(); // ignore: annotate_overrides - - static $pb.PbList createRepeated() => $pb.PbList(); - - static FileStorage getDefault() => _defaultInstance ??= create()..freeze(); - - static FileStorage _defaultInstance; - - $core.int get version => $_get(0, 0); - set version($core.int v) { $_setSignedInt32(0, v); } - $core.bool hasVersion() => $_has(0); - void clearVersion() => clearField(1); - - $core.List get files => $_getList(1); -} diff --git a/packages/flutter_tools/lib/src/build_system/filecache.pbjson.dart b/packages/flutter_tools/lib/src/build_system/filecache.pbjson.dart deleted file mode 100644 index 32f6f9eafc..0000000000 --- a/packages/flutter_tools/lib/src/build_system/filecache.pbjson.dart +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -/// -// Generated code. Do not modify. -// source: lib/src/build_system/filecache.proto -/// -// ignore_for_file: camel_case_types,non_constant_identifier_names,library_prefixes,unused_import,unused_shown_name - -const Map FileHash$json = { - '1': 'FileHash', - '2': >[ - {'1': 'path', '3': 1, '4': 1, '5': 9, '10': 'path'}, - {'1': 'hash', '3': 2, '4': 1, '5': 9, '10': 'hash'}, - ], -}; - -const Map FileStorage$json = { - '1': 'FileHashStore', - '2': >[ - {'1': 'version', '3': 1, '4': 1, '5': 5, '10': 'version'}, - {'1': 'files', '3': 2, '4': 3, '5': 11, '6': '.flutter_tools.FileHash', '10': 'files'}, - ], -}; diff --git a/packages/flutter_tools/lib/src/build_system/filecache.proto b/packages/flutter_tools/lib/src/build_system/filecache.proto deleted file mode 100644 index 63be878f4f..0000000000 --- a/packages/flutter_tools/lib/src/build_system/filecache.proto +++ /dev/null @@ -1,18 +0,0 @@ -syntax = "proto3"; -package flutter_tools; - -message FileHash { - // The absolute path to the file on disk. - string path = 1; - - // The last computed file hash. - string hash = 2; -} - -message FileStorage { - // The current version of the file store. - int32 version = 1; - - // All currently stored files. - repeated FileHash files = 2; -} diff --git a/packages/flutter_tools/lib/src/commands/update_packages.dart b/packages/flutter_tools/lib/src/commands/update_packages.dart index 764fa8611b..a441bc4638 100644 --- a/packages/flutter_tools/lib/src/commands/update_packages.dart +++ b/packages/flutter_tools/lib/src/commands/update_packages.dart @@ -27,7 +27,6 @@ const Map _kManuallyPinnedDependencies = { 'test_api': '0.2.5', // | 'test_core': '0.2.5', // | 'vm_service_client': '0.2.6+2', // Final version before being marked deprecated. - 'protobuf': '0.13.15' // TODO(jonahwilliams): remove usage and update. }; class UpdatePackagesCommand extends FlutterCommand { diff --git a/packages/flutter_tools/pubspec.yaml b/packages/flutter_tools/pubspec.yaml index d1fda625b0..516c5f23e1 100644 --- a/packages/flutter_tools/pubspec.yaml +++ b/packages/flutter_tools/pubspec.yaml @@ -38,7 +38,6 @@ dependencies: yaml: 2.1.16 flutter_goldens_client: path: ../flutter_goldens_client - protobuf: 0.13.15 # We depend on very specific internal implementation details of the # 'test' package, which change between versions, so when upgrading @@ -59,7 +58,7 @@ dependencies: ansi_up: 0.0.1+4.0.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" ansicolor: 1.0.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" async: 2.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" - bazel_worker: 0.1.21 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + bazel_worker: 0.1.22 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" boolean_selector: 1.0.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" browser_launcher: 0.1.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" build_config: 0.4.1+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" @@ -97,6 +96,7 @@ dependencies: polymer_css: 0.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" pool: 1.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" primer_css: 0.0.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + protobuf: 0.14.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" pub_semver: 1.4.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" pubspec_parse: 0.1.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" scratch_space: 0.0.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" @@ -136,4 +136,4 @@ dartdoc: # Exclude this package from the hosted API docs. nodoc: true -# PUBSPEC CHECKSUM: 5e9a +# PUBSPEC CHECKSUM: f467 diff --git a/packages/flutter_tools/test/general.shard/build_system/filecache_test.dart b/packages/flutter_tools/test/general.shard/build_system/filecache_test.dart index 15c7871eeb..7b08bdb357 100644 --- a/packages/flutter_tools/test/general.shard/build_system/filecache_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/filecache_test.dart @@ -5,7 +5,6 @@ import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/build_system/build_system.dart'; import 'package:flutter_tools/src/build_system/file_hash_store.dart'; -import 'package:flutter_tools/src/build_system/filecache.pb.dart' as pb; import '../../src/common.dart'; import '../../src/testbed.dart'; @@ -34,10 +33,10 @@ void main() { final List buffer = fs.file(fs.path.join(environment.buildDir.path, '.filecache')) .readAsBytesSync(); - final pb.FileStorage fileStorage = pb.FileStorage.fromBuffer(buffer); + final FileStorage fileStorage = FileStorage.fromBuffer(buffer); expect(fileStorage.files, isEmpty); - expect(fileStorage.version, 1); + expect(fileStorage.version, 2); })); test('saves and restores to file cache', () => testbed.run(() { @@ -51,7 +50,7 @@ void main() { final String currentHash = fileCache.currentHashes[file.resolveSymbolicLinksSync()]; final List buffer = fs.file(fs.path.join(environment.buildDir.path, '.filecache')) .readAsBytesSync(); - pb.FileStorage fileStorage = pb.FileStorage.fromBuffer(buffer); + FileStorage fileStorage = FileStorage.fromBuffer(buffer); expect(fileStorage.files.single.hash, currentHash); expect(fileStorage.files.single.path, file.resolveSymbolicLinksSync()); @@ -64,7 +63,7 @@ void main() { newFileCache.persist(); // Still persisted correctly. - fileStorage = pb.FileStorage.fromBuffer(buffer); + fileStorage = FileStorage.fromBuffer(buffer); expect(fileStorage.files.single.hash, currentHash); expect(fileStorage.files.single.path, file.resolveSymbolicLinksSync());