mosuem · mosuem · Jul 25, 2023 · Jul 25, 2023 · Jul 25, 2023 · Jul 25, 2023
diff --git a/bin/dart_triage_updater.dart b/bin/dart_triage_updater.dart
@@ -46,5 +46,5 @@ Future<void> main(List<String> arguments) async {
   final updateTypes = toUpdate
       .map((e) => UpdateType.values.firstWhere((type) => type.name == e))
       .toList();
-  await updateThese(updateTypes, github);
+  await TriageUpdater(github).updateThese(updateTypes);
 }
diff --git a/lib/dart_triage_updater.dart b/lib/dart_triage_updater.dart
@@ -1,6 +1,7 @@
 import 'dart:async';
 import 'dart:convert';
 
+import 'package:dart_triage_updater/data_diff.dart';
 import 'package:github/github.dart';
 
 import 'firebase_database.dart';
@@ -10,140 +11,162 @@ import 'repos.dart';
 import 'update_type.dart';
 import 'updater.dart';
 
-Future<void> updateThese(List<UpdateType> updateTypes, GitHub github) async {
-  final updater = Updater();
-  updater.text.stream.listen((event) => print(event));
-  if (updateTypes.contains(UpdateType.issues)) {
-    await update(UpdateType.issues, github, updater, saveAllIssues);
-  }
-  if (updateTypes.contains(UpdateType.pullrequests)) {
-    await update(UpdateType.pullrequests, github, updater, saveAllPullrequests);
-  }
-  if (updateTypes.contains(UpdateType.googlers)) {
-    await updateGooglers(github, updater);
-  }
-}
+class TriageUpdater {
+  final GitHub github;
+  final Updater updater;
 
-Future<void> updateGooglers(GitHub github, Updater updater) async {
-  updater.open('Fetch googlers');
-  final googlersGoogle =
-      await github.organizations.listUsers('google').toList();
-  updater.set('Fetched ${googlersGoogle.length} googlers from "google"');
-  final googlersDart =
-      await github.organizations.listUsers('dart-lang').toList();
-  updater.set('Fetched ${googlersDart.length} googlers from "dart-lang"');
-  final googlers = (googlersGoogle + googlersDart).toSet().toList();
-  updater.set('Store googlers in database');
-  final jsonEncode2 = jsonEncode(googlers);
-  await DatabaseReference.saveGooglers(jsonEncode2);
-  updater.set('Done!');
-  updater.close();
-}
+  TriageUpdater(this.github) : updater = Updater();
 
-Future<void> update(
-  UpdateType type,
-  GitHub github,
-  Updater updater,
-  Future<void> Function(GitHub, RepositorySlug, DatabaseReference, Updater)
-      saveToDatabase,
-) async {
-  final dateTime = DateTime.now();
+  Future<void> updateThese(List<UpdateType> updateTypes) async {
+    updater.text.stream.listen((event) => print(event));
+    if (updateTypes.contains(UpdateType.issues)) {
+      final issuesByRepo = await DatabaseReference.getData(
+        UpdateType.issues,
+        (initial, changes) =>
+            DataDiff(initial, changes, Issue.fromJson).applied()!,
+      );
+      await update(
+        UpdateType.issues,
+        saveAllIssues,
+        issuesByRepo,
+      );
+    }
+    if (updateTypes.contains(UpdateType.pullrequests)) {
+      final pullrequestsByRepo = await DatabaseReference.getData(
+        UpdateType.pullrequests,
+        (initial, changes) =>
+            DataDiff(initial, changes, PullRequest.fromJson).applied()!,
+      );
+      await update(
+        UpdateType.pullrequests,
+        saveAllPullrequests,
+        pullrequestsByRepo,
+      );
+    }
+    if (updateTypes.contains(UpdateType.googlers)) {
+      await updateGooglers();
+    }
+  }
+
+  Future<void> updateGooglers() async {
+    updater.open('Fetch googlers');
+    final googlersGoogle =
+        await github.organizations.listUsers('google').toList();
+    updater.set('Fetched ${googlersGoogle.length} googlers from "google"');
+    final googlersDart =
+        await github.organizations.listUsers('dart-lang').toList();
+    updater.set('Fetched ${googlersDart.length} googlers from "dart-lang"');
+    final googlers = (googlersGoogle + googlersDart).toSet().toList();
+    updater.set('Store googlers in database');
+    final jsonEncode2 = jsonEncode(googlers);
+    await DatabaseReference.saveGooglers(jsonEncode2);
+    updater.set('Done!');
+    updater.close();
+  }
 
-  updater.status.add(true);
+  Future<void> update<T>(
+    UpdateType type,
+    Future<void> Function(
+      RepositorySlug,
+      DatabaseReference,
+      List<T>,
+    ) saveToDatabase,
+    Map<RepositorySlug, List<T>> items,
+  ) async {
+    updater.status.add(true);
 
-  final repositories =
-      github.repositories.listOrganizationRepositories('dart-lang');
-  final dartLangRepos = await repositories
-      .where((repository) => !repository.archived)
-      .map((repository) => repository.slug())
-      .where((slug) => !exludeRepos.contains(slug))
-      .toList();
-  for (final slug in [...dartLangRepos, ...includeRepos]) {
-    try {
-      final ref = DatabaseReference(type, slug);
-      final lastUpdated = await ref.getLastUpdated();
-      final daysSinceUpdate = DateTime.now().difference(lastUpdated).inDays;
-      if (daysSinceUpdate > -1) {
-        ref.setLastUpdated(dateTime);
+    final repositories =
+        github.repositories.listOrganizationRepositories('dart-lang');
+    final dartLangRepos = await repositories
+        .where((repository) => !repository.archived)
+        .map((repository) => repository.slug())
+        .where((slug) => !exludeRepos.contains(slug))
+        .toList();
+    for (final slug in [...dartLangRepos, ...includeRepos]) {
+      try {
+        final ref = DatabaseReference(type, slug);
         final status =
             'Get $type for ${slug.fullName} with ${github.rateLimitRemaining} '
             'remaining requests';
-        ref.deleteAllData();
 
         updater.set(status);
-        await saveToDatabase(github, slug, ref, updater);
-      } else {
-        final status =
-            'Not updating ${slug.fullName} has been updated $daysSinceUpdate '
-            'days ago';
-        updater.set(status);
+        await saveToDatabase(slug, ref, items[slug] ?? []);
+      } catch (e) {
+        updater.set(e.toString());
       }
-    } catch (e) {
-      updater.set(e.toString());
     }
-  }
-
-  updater.close();
-}
 
-Future<void> saveAllPullrequests(
-  GitHub github,
-  RepositorySlug slug,
-  DatabaseReference ref,
-  Updater updater,
-) async {
-  await github.pullRequests.list(slug, pages: 1000).forEach((pr) async {
-    final list = await getReviewers(github, slug, pr);
-    pr.reviewers = list;
-    await addPullRequestToDatabase(pr, ref, updater.text.sink);
-  });
-}
-
-Future<List<User>> getReviewers(
-  GitHub github,
-  RepositorySlug slug,
-  PullRequest pr,
-) async {
-  final reviewers = await github.pullRequests
-      .listReviews(slug, pr.number!)
-      .map((prReview) => prReview.user)
-      .toList();
-  // Deduplicate reviewers
-  final uniqueNames = reviewers.map((e) => e.login).whereType<String>().toSet();
-  reviewers.retainWhere((reviewer) => uniqueNames.remove(reviewer.login));
-  return reviewers;
-}
+    updater.close();
+  }
 
-Future<void> saveAllIssues(
-  GitHub github,
-  RepositorySlug slug,
-  DatabaseReference ref,
-  Updater updater,
-) async {
-  await github.issues.listByRepo(slug, perPage: 1000).forEach((pr) async {
-    if (pr.pullRequest == null) {
-      await addIssueToDatabase(pr, ref, updater.text.sink);
+  Future<void> saveAllPullrequests(
+    RepositorySlug slug,
+    DatabaseReference ref,
+    List<PullRequest> pullrequests,
+  ) async {
+    await github.pullRequests
+        .list(slug, pages: 1000)
+        .forEach((pullrequest) async {
+      final oldPullrequest =
+          pullrequests.where((pr) => pr.id == pullrequest.id).firstOrNull;
+      final reviewers = await getReviewers(github, slug, pullrequest);
+      pullrequest.reviewers = reviewers;
+      await DatabaseReference.addChange(
+        UpdateType.pullrequests,
+        pullrequest.id.toString(),
+        oldPullrequest,
+        pullrequest,
+      );
+    });
+    for (final remainingPr in pullrequests) {
+      await DatabaseReference.addChange(
+        UpdateType.pullrequests,
+        remainingPr.id.toString(),
+        remainingPr,
+        remainingPr.close(),
+      );
     }
-  });
-}
+  }
 
-Future<void> addPullRequestToDatabase(
-  PullRequest pr,
-  DatabaseReference ref, [
-  StreamSink<String?>? logger,
-]) async {
-  logger?.add('\tHandle PR ${pr.id} from ${pr.base!.repo!.slug().fullName}');
-  final jsonEncode2 = jsonEncode({pr.id!.toString(): encodePR(pr)});
-  return await ref.addData(jsonEncode2);
-}
+  Future<void> saveAllIssues(
+    RepositorySlug slug,
+    DatabaseReference ref,
+    List<Issue> issues,
+  ) async {
+    await github.issues.listByRepo(slug, perPage: 1000).forEach((issue) async {
+      final oldIssue =
+          issues.where((element) => element.id == issue.id).firstOrNull;
+      await DatabaseReference.addChange(
+        UpdateType.issues,
+        issue.id.toString(),
+        oldIssue,
+        issue,
+      );
+      if (oldIssue != null) issues.remove(oldIssue);
+    });
+    for (final remainingIssue in issues) {
+      await DatabaseReference.addChange(
+        UpdateType.issues,
+        remainingIssue.id.toString(),
+        remainingIssue,
+        remainingIssue.close(),
+      );
+    }
+  }
 
-Future<void> addIssueToDatabase(
-  Issue issue,
-  DatabaseReference ref, [
-  StreamSink<String?>? logger,
-]) async {
-  logger?.add(
-      '\tHandle Issue ${issue.id} from ${issue.repositoryUrl?.substring('https://api.github.com/repos/'.length)}');
-  return await ref
-      .addData(jsonEncode({issue.id.toString(): encodeIssue(issue)}));
+  Future<List<User>> getReviewers(
+    GitHub github,
+    RepositorySlug slug,
+    PullRequest pr,
+  ) async {
+    final reviewers = await github.pullRequests
+        .listReviews(slug, pr.number!)
+        .map((prReview) => prReview.user)
+        .toList();
+    // Deduplicate reviewers
+    final uniqueNames =
+        reviewers.map((e) => e.login).whereType<String>().toSet();
+    reviewers.retainWhere((reviewer) => uniqueNames.remove(reviewer.login));
+    return reviewers;
+  }
 }
diff --git a/lib/data_diff.dart b/lib/data_diff.dart
@@ -0,0 +1,45 @@
+import 'package:dart_triage_updater/differ.dart';
+import 'package:json_diff/json_diff.dart';
+
+class DataDiff<S> {
+  final Map<String, dynamic> _initial;
+  final Map<DateTime, DiffNode> _changes;
+  final S Function(Map<String, dynamic> json) fromJson;
+
+  DataDiff(this._initial, this._changes, this.fromJson) {
+    assert(_changes.values.first.toJsonString() == DiffNode([]).toJsonString());
+  }
+
+  S? applied([DateTime? atTime]) {
+    final map = _changes.entries
+        .where((entry) => atTime != null ? entry.key.isBefore(atTime) : true)
+        .map((e) => e.value);
+    if (map.isNotEmpty) {
+      final allChanges = map.reduce((node1, node2) => node1 + node2);
+      return fromJson(allChanges.apply(_initial));
+    }
+    return null;
+  }
+
+  Map<DateTime, T> getTimeSeries<T>(T Function(S issue) getData) {
+    var current = _initial;
+    final result = <DateTime, T>{};
+    for (final element in _changes.entries) {
+      current = element.value.apply(current);
+      result[element.key] = getData(fromJson(current));
+    }
+    return result;
+  }
+
+  Duration? getTimeUntil(bool Function(S) condition) {
+    var current = _initial;
+    final initialEntry = _changes.entries.first.key;
+    for (final element in _changes.entries) {
+      current = element.value.apply(current);
+      if (condition(fromJson(current))) {
+        return element.key.difference(initialEntry);
+      }
+    }
+    return null;
+  }
+}