refact: optimize, ID search peers (#10853)

* refact: optimize, preload peers

Signed-off-by: fufesou <linlong1266@gmail.com>

* Update dialogs.dart

---------

Signed-off-by: fufesou <linlong1266@gmail.com>
Co-authored-by: RustDesk <71636191+rustdesk@users.noreply.github.com>

flutter/lib/common.dart

@@ -822,7 +822,11 @@ class OverlayDialogManager {
 
     close([res]) {
       _dialogs.remove(dialogTag);
-      dialog.complete(res);
+      try {
+        dialog.complete(res);
+      } catch (e) {
+        debugPrint("Dialog complete catch error: $e");
+      }
       BackButtonInterceptor.removeByName(dialogTag);
     }
 

flutter/lib/common/widgets/autocomplete.dart

@@ -1,4 +1,3 @@
-import 'dart:convert';
 import 'package:flutter/material.dart';
 import 'package:flutter_hbb/common/formatter/id_formatter.dart';
 import '../../../models/platform_model.dart';
@@ -8,87 +7,56 @@ import 'package:flutter_hbb/common/widgets/peer_card.dart';
 
 class AllPeersLoader {
   List<Peer> peers = [];
-  bool hasMoreRecentPeers = false;
 
   bool isPeersLoading = false;
-  bool _isPartialPeersLoaded = false;
+  bool isPeersLoaded = false;
-  bool _isPeersLoaded = false;
+
+  final String _listenerKey = 'AllPeersLoader';
+
+  late void Function(VoidCallback) setState;
 
   AllPeersLoader();
 
-  bool get isLoaded => _isPartialPeersLoaded || _isPeersLoaded;
+  void init(void Function(VoidCallback) setState) {
-
+    this.setState = setState;
-  void reset() {
+    gFFI.recentPeersModel.addListener(_mergeAllPeers);
-    peers.clear();
+    gFFI.lanPeersModel.addListener(_mergeAllPeers);
-    hasMoreRecentPeers = false;
+    gFFI.abModel.addPeerUpdateListener(_listenerKey, _mergeAllPeers);
-    _isPartialPeersLoaded = false;
+    gFFI.groupModel.addPeerUpdateListener(_listenerKey, _mergeAllPeers);
-    _isPeersLoaded = false;
   }
 
-  Future<void> getAllPeers(void Function(VoidCallback) setState) async {
+  void clear() {
-    if (isPeersLoading) {
+    gFFI.recentPeersModel.removeListener(_mergeAllPeers);
+    gFFI.lanPeersModel.removeListener(_mergeAllPeers);
+    gFFI.abModel.removePeerUpdateListener(_listenerKey);
+    gFFI.groupModel.removePeerUpdateListener(_listenerKey);
+  }
+
+  Future<void> getAllPeers() async {
+    if (isPeersLoaded || isPeersLoading) {
       return;
     }
-    reset();
     isPeersLoading = true;
 
+    if (gFFI.recentPeersModel.peers.isEmpty) {
+      bind.mainLoadRecentPeers();
+    }
+    if (gFFI.lanPeersModel.peers.isEmpty) {
+      bind.mainLoadLanPeers();
+    }
+    // No need to care about peers from abModel, and group model.
+    // Because they will pull data in `refreshCurrentUser()` on startup.
+
     final startTime = DateTime.now();
-    await _getAllPeers(false);
+    _mergeAllPeers();
-    if (!hasMoreRecentPeers) {
+    final diffTime = DateTime.now().difference(startTime).inMilliseconds;
-      final diffTime = DateTime.now().difference(startTime).inMilliseconds;
+    if (diffTime < 100) {
-      if (diffTime < 100) {
+      await Future.delayed(Duration(milliseconds: diffTime));
-        await Future.delayed(Duration(milliseconds: diffTime));
-      }
-      setState(() {
-        isPeersLoading = false;
-        _isPeersLoaded = true;
-      });
-    } else {
-      setState(() {
-        _isPartialPeersLoaded = true;
-      });
-      await _getAllPeers(true);
-      setState(() {
-        isPeersLoading = false;
-        _isPeersLoaded = true;
-      });
     }
   }
 
-  Future<void> _getAllPeers(bool getAllRecentPeers) async {
+  void _mergeAllPeers() {
-    Map<String, dynamic> recentPeers =
-        jsonDecode(await bind.mainGetRecentPeers(getAll: getAllRecentPeers));
-    Map<String, dynamic> lanPeers = jsonDecode(bind.mainLoadLanPeersSync());
     Map<String, dynamic> combinedPeers = {};
-
-    void mergePeers(Map<String, dynamic> peers) {
-      if (peers.containsKey("peers")) {
-        dynamic peerData = peers["peers"];
-
-        if (peerData is String) {
-          try {
-            peerData = jsonDecode(peerData);
-          } catch (e) {
-            print("Error decoding peers: $e");
-            return;
-          }
-        }
-
-        if (peerData is List) {
-          for (var peer in peerData) {
-            if (peer is Map && peer.containsKey("id")) {
-              String id = peer["id"];
-              if (!combinedPeers.containsKey(id)) {
-                combinedPeers[id] = peer;
-              }
-            }
-          }
-        }
-      }
-    }
-
-    mergePeers(recentPeers);
-    mergePeers(lanPeers);
     for (var p in gFFI.abModel.allPeers()) {
       if (!combinedPeers.containsKey(p.id)) {
         combinedPeers[p.id] = p.toJson();
@@ -101,27 +69,36 @@ class AllPeersLoader {
     }
 
     List<Peer> parsedPeers = [];
-
     for (var peer in combinedPeers.values) {
       parsedPeers.add(Peer.fromJson(peer));
     }
 
-    try {
+    Set<String> peerIds = combinedPeers.keys.toSet();
-      final List<dynamic> moreRecentPeerIds =
+    for (final peer in gFFI.lanPeersModel.peers) {
-          jsonDecode(recentPeers["ids"] ?? '[]');
+      if (!peerIds.contains(peer.id)) {
-      hasMoreRecentPeers = false;
+        parsedPeers.add(peer);
-      for (final id in moreRecentPeerIds) {
+        peerIds.add(peer.id);
-        final sid = id.toString();
+      }
-        if (!parsedPeers.any((element) => element.id == sid)) {
+    }
-          parsedPeers.add(Peer.fromJson({'id': sid}));
+
-          hasMoreRecentPeers = true;
+    for (final peer in gFFI.recentPeersModel.peers) {
-        }
+      if (!peerIds.contains(peer.id)) {
+        parsedPeers.add(peer);
+        peerIds.add(peer.id);
+      }
+    }
+    for (final id in gFFI.recentPeersModel.restPeerIds) {
+      if (!peerIds.contains(id)) {
+        parsedPeers.add(Peer.fromJson({'id': id}));
+        peerIds.add(id);
       }
-    } catch (e) {
-      debugPrint("Error parsing more peer ids: $e");
     }
 
     peers = parsedPeers;
+    setState(() {
+      isPeersLoading = false;
+      isPeersLoaded = true;
+    });
   }
 }
 

flutter/lib/common/widgets/peer_card.dart

@@ -716,18 +716,18 @@ abstract class BasePeerCard extends StatelessWidget {
           switch (tab) {
             case PeerTabIndex.recent:
               await bind.mainRemovePeer(id: id);
-              await bind.mainLoadRecentPeers();
+              bind.mainLoadRecentPeers();
               break;
             case PeerTabIndex.fav:
               final favs = (await bind.mainGetFav()).toList();
               if (favs.remove(id)) {
                 await bind.mainStoreFav(favs: favs);
-                await bind.mainLoadFavPeers();
+                bind.mainLoadFavPeers();
               }
               break;
             case PeerTabIndex.lan:
               await bind.mainRemoveDiscovered(id: id);
-              await bind.mainLoadLanPeers();
+              bind.mainLoadLanPeers();
               break;
             case PeerTabIndex.ab:
               await gFFI.abModel.deletePeers([id]);

flutter/lib/common/widgets/peer_tab_page.dart

@@ -404,7 +404,7 @@ class _PeerTabPageState extends State<PeerTabPage>
                 for (var p in peers) {
                   await bind.mainRemovePeer(id: p.id);
                 }
-                await bind.mainLoadRecentPeers();
+                bind.mainLoadRecentPeers();
                 break;
               case 1:
                 final favs = (await bind.mainGetFav()).toList();
@@ -412,13 +412,13 @@ class _PeerTabPageState extends State<PeerTabPage>
                   favs.remove(p.id);
                 }).toList();
                 await bind.mainStoreFav(favs: favs);
-                await bind.mainLoadFavPeers();
+                bind.mainLoadFavPeers();
                 break;
               case 2:
                 for (var p in peers) {
                   await bind.mainRemoveDiscovered(id: p.id);
                 }
-                await bind.mainLoadLanPeers();
+                bind.mainLoadLanPeers();
                 break;
               case 3:
                 await gFFI.abModel.deletePeers(peers.map((p) => p.id).toList());

flutter/lib/desktop/pages/connection_page.dart

@@ -205,7 +205,7 @@ class _ConnectionPageState extends State<ConnectionPage>
 
   bool isWindowMinimized = false;
 
-  AllPeersLoader allPeersLoader = AllPeersLoader();
+  final AllPeersLoader _allPeersLoader = AllPeersLoader();
 
   // https://github.com/flutter/flutter/issues/157244
   Iterable<Peer> _autocompleteOpts = [];
@@ -213,6 +213,7 @@ class _ConnectionPageState extends State<ConnectionPage>
   @override
   void initState() {
     super.initState();
+    _allPeersLoader.init(setState);
     _idFocusNode.addListener(onFocusChanged);
     if (_idController.text.isEmpty) {
       WidgetsBinding.instance.addPostFrameCallback((_) async {
@@ -232,6 +233,7 @@ class _ConnectionPageState extends State<ConnectionPage>
   void dispose() {
     _idController.dispose();
     windowManager.removeListener(this);
+    _allPeersLoader.clear();
     _idFocusNode.removeListener(onFocusChanged);
     _idFocusNode.dispose();
     _idEditingController.dispose();
@@ -280,8 +282,8 @@ class _ConnectionPageState extends State<ConnectionPage>
 
   void onFocusChanged() {
     _idInputFocused.value = _idFocusNode.hasFocus;
-    if (_idFocusNode.hasFocus && !allPeersLoader.isPeersLoading) {
+    if (_idFocusNode.hasFocus && !_allPeersLoader.isPeersLoading) {
-      allPeersLoader.getAllPeers(setState);
+      _allPeersLoader.getAllPeers();
     }
   }
 
@@ -336,8 +338,8 @@ class _ConnectionPageState extends State<ConnectionPage>
                   optionsBuilder: (TextEditingValue textEditingValue) {
                     if (textEditingValue.text == '') {
                       _autocompleteOpts = const Iterable<Peer>.empty();
-                    } else if (allPeersLoader.peers.isEmpty &&
+                    } else if (_allPeersLoader.peers.isEmpty &&
-                        !allPeersLoader.isLoaded) {
+                        !_allPeersLoader.isPeersLoaded) {
                       Peer emptyPeer = Peer(
                         id: '',
                         username: '',
@@ -364,7 +366,7 @@ class _ConnectionPageState extends State<ConnectionPage>
                         );
                       }
                       String textToFind = textEditingValue.text.toLowerCase();
-                      _autocompleteOpts = allPeersLoader.peers
+                      _autocompleteOpts = _allPeersLoader.peers
                           .where((peer) =>
                               peer.id.toLowerCase().contains(textToFind) ||
                               peer.username
@@ -471,8 +473,8 @@ class _ConnectionPageState extends State<ConnectionPage>
                                     maxHeight: maxHeight,
                                     maxWidth: 319,
                                   ),
-                                  child: allPeersLoader.peers.isEmpty &&
+                                  child: _allPeersLoader.peers.isEmpty &&
-                                          !allPeersLoader.isLoaded
+                                          !_allPeersLoader.isPeersLoaded
                                       ? Container(
                                           height: 80,
                                           child: Center(

flutter/lib/mobile/pages/connection_page.dart

@@ -44,7 +44,7 @@ class _ConnectionPageState extends State<ConnectionPage> {
   final FocusNode _idFocusNode = FocusNode();
   final TextEditingController _idEditingController = TextEditingController();
 
-  AllPeersLoader allPeersLoader = AllPeersLoader();
+  final AllPeersLoader _allPeersLoader = AllPeersLoader();
 
   StreamSubscription? _uniLinksSubscription;
 
@@ -62,6 +62,7 @@ class _ConnectionPageState extends State<ConnectionPage> {
   @override
   void initState() {
     super.initState();
+    _allPeersLoader.init(setState);
     _idFocusNode.addListener(onFocusChanged);
     if (_idController.text.isEmpty) {
       WidgetsBinding.instance.addPostFrameCallback((_) async {
@@ -103,8 +104,8 @@ class _ConnectionPageState extends State<ConnectionPage> {
 
   void onFocusChanged() {
     _idEmpty.value = _idEditingController.text.isEmpty;
-    if (_idFocusNode.hasFocus && !allPeersLoader.isPeersLoading) {
+    if (_idFocusNode.hasFocus && !_allPeersLoader.isPeersLoading) {
-      allPeersLoader.getAllPeers(setState);
+      _allPeersLoader.getAllPeers();
     }
   }
 
@@ -157,8 +158,8 @@ class _ConnectionPageState extends State<ConnectionPage> {
                     optionsBuilder: (TextEditingValue textEditingValue) {
                       if (textEditingValue.text == '') {
                         _autocompleteOpts = const Iterable<Peer>.empty();
-                      } else if (allPeersLoader.peers.isEmpty &&
+                      } else if (_allPeersLoader.peers.isEmpty &&
-                          !allPeersLoader.isLoaded) {
+                          !_allPeersLoader.isPeersLoaded) {
                         Peer emptyPeer = Peer(
                           id: '',
                           username: '',
@@ -186,7 +187,7 @@ class _ConnectionPageState extends State<ConnectionPage> {
                         }
                         String textToFind = textEditingValue.text.toLowerCase();
 
-                        _autocompleteOpts = allPeersLoader.peers
+                        _autocompleteOpts = _allPeersLoader.peers
                             .where((peer) =>
                                 peer.id.toLowerCase().contains(textToFind) ||
                                 peer.username
@@ -296,8 +297,9 @@ class _ConnectionPageState extends State<ConnectionPage> {
                                             maxHeight: maxHeight,
                                             maxWidth: 320,
                                           ),
-                                          child: allPeersLoader.peers.isEmpty &&
+                                          child: _allPeersLoader
-                                                  !allPeersLoader.isLoaded
+                                                      .peers.isEmpty &&
+                                                  !_allPeersLoader.isPeersLoaded
                                               ? Container(
                                                   height: 80,
                                                   child: Center(
@@ -361,6 +363,7 @@ class _ConnectionPageState extends State<ConnectionPage> {
     _uniLinksSubscription?.cancel();
     _idController.dispose();
     _idFocusNode.removeListener(onFocusChanged);
+    _allPeersLoader.clear();
     _idFocusNode.dispose();
     _idEditingController.dispose();
     if (Get.isRegistered<IDTextEditingController>()) {

flutter/lib/models/ab_model.dart

@@ -58,6 +58,9 @@ class AbModel {
   String? _personalAbGuid;
   RxBool legacyMode = false.obs;
 
+  // Only handles peers add/remove
+  final Map<String, VoidCallback> _peerIdUpdateListeners = {};
+
   final sortTags = shouldSortTags().obs;
   final filterByIntersection = filterAbTagByIntersection().obs;
 
@@ -188,6 +191,7 @@ class AbModel {
         debugPrint("pull current Ab error: $e");
       }
     }
+    _callbackPeerUpdate();
     if (listInitialized && current.initialized) {
       _saveCache();
     }
@@ -419,6 +423,7 @@ class AbModel {
         }
       });
     }
+    _callbackPeerUpdate();
     return ret;
   }
 
@@ -620,6 +625,9 @@ class AbModel {
           }
         }
       }
+      if (abEntries.isNotEmpty) {
+        _callbackPeerUpdate();
+      }
     }
   }
 
@@ -742,6 +750,20 @@ class AbModel {
     }
   }
 
+  void _callbackPeerUpdate() {
+    for (var listener in _peerIdUpdateListeners.values) {
+      listener();
+    }
+  }
+
+  void addPeerUpdateListener(String key, VoidCallback listener) {
+    _peerIdUpdateListeners[key] = listener;
+  }
+
+  void removePeerUpdateListener(String key) {
+    _peerIdUpdateListeners.remove(key);
+  }
+
 // #endregion
 }
 

flutter/lib/models/group_model.dart

@@ -23,6 +23,8 @@ class GroupModel {
   var _cacheLoadOnceFlag = false;
   var _statusCode = 200;
 
+  final Map<String, VoidCallback> _peerIdUpdateListeners = {};
+
   bool get emtpy => deviceGroups.isEmpty && users.isEmpty && peers.isEmpty;
 
   late final Peers peersModel;
@@ -92,6 +94,7 @@ class GroupModel {
         .map((e) => e.online = true)
         .toList();
     groupLoadError.value = '';
+    _callbackPeerUpdate();
   }
 
   Future<bool> _getDeviceGroups(
@@ -329,6 +332,7 @@ class GroupModel {
         for (final peer in data['peers']) {
           peers.add(Peer.fromJson(peer));
         }
+        _callbackPeerUpdate();
       }
     } catch (e) {
       debugPrint("load group cache: $e");
@@ -343,4 +347,18 @@ class GroupModel {
     selectedAccessibleItemName.value = '';
     await bind.mainClearGroup();
   }
+
+  void _callbackPeerUpdate() {
+    for (var listener in _peerIdUpdateListeners.values) {
+      listener();
+    }
+  }
+
+  void addPeerUpdateListener(String key, VoidCallback listener) {
+    _peerIdUpdateListeners[key] = listener;
+  }
+
+  void removePeerUpdateListener(String key) {
+    _peerIdUpdateListeners.remove(key);
+  }
 }

flutter/lib/models/peer_model.dart

@@ -165,6 +165,11 @@ class Peers extends ChangeNotifier {
   final String name;
   final String loadEvent;
   List<Peer> peers = List.empty(growable: true);
+  // Part of the peers that are not in the rest peers list.
+  // When there're too many peers, we may want to load the front 100 peers first,
+  // so we can see peers in UI quickly. `restPeerIds` is the rest peers' ids.
+  // And then load all peers later.
+  List<String> restPeerIds = List.empty(growable: true);
   final GetInitPeers? getInitPeers;
   UpdateEvent event = UpdateEvent.load;
   static const _cbQueryOnlines = 'callback_query_onlines';
@@ -238,6 +243,12 @@ class Peers extends ChangeNotifier {
     } else {
       peers = _decodePeers(evt['peers']);
     }
+
+    restPeerIds = [];
+    if (evt['ids'] != null) {
+      restPeerIds = (evt['ids'] as String).split(',');
+    }
+
     for (var peer in peers) {
       final state = onlineStates[peer.id];
       peer.online = state != null && state != false;

flutter/lib/plugin/utils/dialogs.dart

@@ -2,14 +2,15 @@ import 'dart:convert';
 
 import 'package:flutter/material.dart';
 import 'package:flutter_hbb/common.dart';
-import 'package:flutter_hbb/models/platform_model.dart';
 
 void showPeerSelectionDialog(
     {bool singleSelection = false,
     required Function(List<String>) onPeersCallback}) async {
-  final peers = await bind.mainGetRecentPeers(getAll: true);
+  // load recent peers, we can directly use the peers in `gFFI.recentPeersModel`.
+  // The plugin is not used for now, so just left it empty here.
+  final peers = '';
   if (peers.isEmpty) {
-    debugPrint("load recent peers failed.");
+    // debugPrint("load recent peers failed.");
     return;
   }
 

src/flutter_ffi.rs

@@ -1127,84 +1127,48 @@ pub fn main_load_recent_peers() {
             return;
         }
 
-        let push_to_flutter = |peers| {
+        let push_to_flutter = |peers, ids| {
-            let data = HashMap::from([("name", "load_recent_peers".to_owned()), ("peers", peers)]);
+            let mut data =
+                HashMap::from([("name", "load_recent_peers".to_owned()), ("peers", peers)]);
+            if let Some(ids) = ids {
+                data.insert("ids", ids);
+            }
             let _res = flutter::push_global_event(
                 flutter::APP_TYPE_MAIN,
                 serde_json::ser::to_string(&data).unwrap_or("".to_owned()),
             );
         };
 
-        let load_two_times =
+        let load_two_times = vec_id_modified_time_path.len() > PeerConfig::BATCH_LOADING_COUNT
-            vec_id_modified_time_path.len() > PeerConfig::BATCH_LOADING_COUNT && cfg!(target_os = "windows");
+            && cfg!(target_os = "windows");
         let mut all_peers = vec![];
         if load_two_times {
             let next_from = load_recent_peers(&vec_id_modified_time_path, false, &mut all_peers, 0);
-            push_to_flutter(serde_json::ser::to_string(&all_peers).unwrap_or("".to_owned()));
+            let rest_ids = if next_from < vec_id_modified_time_path.len() {
+                Some(
+                    vec_id_modified_time_path[next_from..]
+                        .iter()
+                        .map(|(id, _, _)| id.clone())
+                        .collect::<Vec<_>>()
+                        .join(", "),
+                )
+            } else {
+                None
+            };
+            push_to_flutter(
+                serde_json::ser::to_string(&all_peers).unwrap_or("".to_owned()),
+                rest_ids,
+            );
             let _ = load_recent_peers(&vec_id_modified_time_path, true, &mut all_peers, next_from);
         } else {
             let _ = load_recent_peers(&vec_id_modified_time_path, true, &mut all_peers, 0);
         }
-        push_to_flutter(serde_json::ser::to_string(&all_peers).unwrap_or("".to_owned()));
+        // Don't check if `all_peers` is empty, because we need this message to update the state in the flutter side.
-    }
+        push_to_flutter(
-}
+            serde_json::ser::to_string(&all_peers).unwrap_or("".to_owned()),
-
+            None,
-fn get_partial_recent_peers(
-    vec_id_modified_time_path: Vec<(String, SystemTime, std::path::PathBuf)>,
-    count: usize,
-) -> String {
-    let (peers, next_from) = PeerConfig::batch_peers(
-        &vec_id_modified_time_path,
-        0,
-        Some(count.min(vec_id_modified_time_path.len())),
-    );
-    let peer_maps: Vec<_> = peers
-        .into_iter()
-        .map(|(id, _, p)| peer_to_map(id, p))
-        .collect();
-    let mut data = HashMap::from([(
-        "peers",
-        serde_json::ser::to_string(&peer_maps).unwrap_or("[]".to_owned()),
-    )]);
-    if next_from < vec_id_modified_time_path.len() {
-        let ids: Vec<_> = vec_id_modified_time_path[next_from..]
-            .iter()
-            .map(|(id, _, _)| id.clone())
-            .collect();
-        data.insert(
-            "ids",
-            serde_json::ser::to_string(&ids).unwrap_or("[]".to_owned()),
         );
     }
-    return serde_json::ser::to_string(&data).unwrap_or("".to_owned());
-}
-
-pub fn main_get_recent_peers(get_all: bool) -> String {
-    if !config::APP_DIR.read().unwrap().is_empty() {
-        let vec_id_modified_time_path = PeerConfig::get_vec_id_modified_time_path(&None);
-
-        let load_two_times = !get_all
-            && vec_id_modified_time_path.len() > PeerConfig::BATCH_LOADING_COUNT
-            && cfg!(target_os = "windows");
-        let load_count = if load_two_times {
-            PeerConfig::BATCH_LOADING_COUNT
-        } else {
-            vec_id_modified_time_path.len()
-        };
-        return get_partial_recent_peers(vec_id_modified_time_path, load_count);
-    }
-    "".to_string()
-}
-
-pub fn main_load_lan_peers_sync() -> SyncReturn<String> {
-    let data = HashMap::from([
-        ("name", "load_lan_peers".to_owned()),
-        (
-            "peers",
-            serde_json::to_string(&get_lan_peers()).unwrap_or_default(),
-        ),
-    ]);
-    return SyncReturn(serde_json::ser::to_string(&data).unwrap_or("".to_owned()));
 }
 
 pub fn main_load_recent_peers_for_ab(filter: String) -> String {