Port of fix for b/67042460 (retry more often before considering client offline) (#403)

This ensures FSTRemoteStore always tries to connect at least twice before surfacing an FSTOnlineStateFailed
event to external code (which may trigger gets to fail, cached data to be surfaced, etc.).

4 files changed, 164 insertions, 84 deletions

diff --git a/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.m b/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.m
index b4c0b02..b83942b 100644
--- a/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.m
+++ b/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.m
@@ -263,8 +263,10 @@ NS_ASSUME_NONNULL_BEGIN
       [NSError errorWithDomain:FIRFirestoreErrorDomain code:errorCode userInfo:userInfo];
 
   [self.datastore failWatchStreamWithError:error];
-  // Unlike web, stream should re-open synchronously
-  FSTAssert(self.datastore.isWatchStreamOpen, @"Watch stream is open");
+  // Unlike web, stream should re-open synchronously (if we have any listeners)
+  if (self.queryListeners.count > 0) {
+    FSTAssert(self.datastore.isWatchStreamOpen, @"Watch stream is open");
+  }
 }
 
 - (NSDictionary<FSTDocumentKey *, FSTBoxedTargetID *> *)currentLimboDocuments {
diff --git a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json
index e58bae1..f542a6e 100644
--- a/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json
+++ b/Firestore/Example/Tests/SpecTests/json/offline_spec_test.json
@@ -35,6 +35,14 @@
             "code": 14,
             "message": "Simulated Backend Error"
           }
+        }
+      },
+      {
+        "watchStreamClose": {
+          "error": {
+            "code": 14,
+            "message": "Simulated Backend Error"
+          }
         },
         "expect": [
           {
@@ -116,6 +124,14 @@
             "code": 14,
             "message": "Simulated Backend Error"
           }
+        }
+      },
+      {
+        "watchStreamClose": {
+          "error": {
+            "code": 14,
+            "message": "Simulated Backend Error"
+          }
         },
         "expect": [
           {
@@ -147,5 +163,136 @@
         }
       }
     ]
+  },
+  "Removing all listeners delays \"Offline\" status on next listen": {
+    "describeName": "Offline:",
+    "itName": "Removing all listeners delays \"Offline\" status on next listen",
+    "tags": [],
+    "config": {
+      "useGarbageCollection": true
+    },
+    "steps": [
+      {
+        "userListen": [
+          2,
+          {
+            "path": "collection",
+            "filters": [],
+            "orderBys": []
+          }
+        ],
+        "stateExpect": {
+          "activeTargets": {
+            "2": {
+              "query": {
+                "path": "collection",
+                "filters": [],
+                "orderBys": []
+              },
+              "resumeToken": ""
+            }
+          }
+        }
+      },
+      {
+        "watchStreamClose": {
+          "error": {
+            "code": 14,
+            "message": "Simulated Backend Error"
+          }
+        }
+      },
+      {
+        "watchStreamClose": {
+          "error": {
+            "code": 14,
+            "message": "Simulated Backend Error"
+          }
+        },
+        "expect": [
+          {
+            "query": {
+              "path": "collection",
+              "filters": [],
+              "orderBys": []
+            },
+            "errorCode": 0,
+            "fromCache": true,
+            "hasPendingWrites": false
+          }
+        ]
+      },
+      {
+        "userUnlisten": [
+          2,
+          {
+            "path": "collection",
+            "filters": [],
+            "orderBys": []
+          }
+        ],
+        "stateExpect": {
+          "activeTargets": {}
+        }
+      },
+      {
+        "watchStreamClose": {
+          "error": {
+            "code": 14,
+            "message": "Simulated Backend Error"
+          }
+        }
+      },
+      {
+        "userListen": [
+          4,
+          {
+            "path": "collection",
+            "filters": [],
+            "orderBys": []
+          }
+        ],
+        "stateExpect": {
+          "activeTargets": {
+            "4": {
+              "query": {
+                "path": "collection",
+                "filters": [],
+                "orderBys": []
+              },
+              "resumeToken": ""
+            }
+          }
+        }
+      },
+      {
+        "watchStreamClose": {
+          "error": {
+            "code": 14,
+            "message": "Simulated Backend Error"
+          }
+        }
+      },
+      {
+        "watchStreamClose": {
+          "error": {
+            "code": 14,
+            "message": "Simulated Backend Error"
+          }
+        },
+        "expect": [
+          {
+            "query": {
+              "path": "collection",
+              "filters": [],
+              "orderBys": []
+            },
+            "errorCode": 0,
+            "fromCache": true,
+            "hasPendingWrites": false
+          }
+        ]
+      }
+    ]
   }
 }
diff --git a/Firestore/Example/Tests/SpecTests/json/remote_store_spec_test.json b/Firestore/Example/Tests/SpecTests/json/remote_store_spec_test.json
index edb0751..26bb520 100644
--- a/Firestore/Example/Tests/SpecTests/json/remote_store_spec_test.json
+++ b/Firestore/Example/Tests/SpecTests/json/remote_store_spec_test.json
@@ -489,19 +489,7 @@
             "code": 14,
             "message": "Simulated Backend Error"
           }
-        },
-        "expect": [
-          {
-            "query": {
-              "path": "collection",
-              "filters": [],
-              "orderBys": []
-            },
-            "errorCode": 0,
-            "fromCache": true,
-            "hasPendingWrites": false
-          }
-        ]
+        }
       },
       {
         "watchAck": [
diff --git a/Firestore/Example/Tests/SpecTests/json/write_spec_test.json b/Firestore/Example/Tests/SpecTests/json/write_spec_test.json
index 60ed107..51d1aee 100644
--- a/Firestore/Example/Tests/SpecTests/json/write_spec_test.json
+++ b/Firestore/Example/Tests/SpecTests/json/write_spec_test.json
@@ -4372,9 +4372,9 @@
       }
     ]
   },
-  "Writes that fail with code cancelled are retried": {
+  "Writes that fail with code resource_exhausted are not rejected": {
     "describeName": "Writes:",
-    "itName": "Writes that fail with code cancelled are retried",
+    "itName": "Writes that fail with code resource_exhausted are not rejected",
     "tags": [],
     "config": {
       "useGarbageCollection": true
@@ -4435,73 +4435,16 @@
       {
         "failWrite": {
           "error": {
-            "code": 1
+            "code": 8
           },
           "expectUserCallback": false
         }
-      },
-      {
-        "writeAck": {
-          "version": 1000,
-          "expectUserCallback": true
-        }
-      },
-      {
-        "watchAck": [
-          2
-        ]
-      },
-      {
-        "watchEntity": {
-          "docs": [
-            [
-              "collection/key",
-              0,
-              {
-                "foo": "bar"
-              }
-            ]
-          ],
-          "targets": [
-            2
-          ]
-        }
-      },
-      {
-        "watchCurrent": [
-          [
-            2
-          ],
-          "resume-token-1000"
-        ],
-        "watchSnapshot": 1000,
-        "expect": [
-          {
-            "query": {
-              "path": "collection/key",
-              "filters": [],
-              "orderBys": []
-            },
-            "metadata": [
-              [
-                "collection/key",
-                0,
-                {
-                  "foo": "bar"
-                }
-              ]
-            ],
-            "errorCode": 0,
-            "fromCache": false,
-            "hasPendingWrites": false
-          }
-        ]
       }
     ]
   },
-  "Writes that fail with code unknown are retried": {
+  "Writes that fail with code cancelled are retried": {
     "describeName": "Writes:",
-    "itName": "Writes that fail with code unknown are retried",
+    "itName": "Writes that fail with code cancelled are retried",
     "tags": [],
     "config": {
       "useGarbageCollection": true
@@ -4562,7 +4505,7 @@
       {
         "failWrite": {
           "error": {
-            "code": 2
+            "code": 1
           },
           "expectUserCallback": false
         }
@@ -4626,9 +4569,9 @@
       }
     ]
   },
-  "Writes that fail with code deadline-exceeded are retried": {
+  "Writes that fail with code unknown are retried": {
     "describeName": "Writes:",
-    "itName": "Writes that fail with code deadline-exceeded are retried",
+    "itName": "Writes that fail with code unknown are retried",
     "tags": [],
     "config": {
       "useGarbageCollection": true
@@ -4689,7 +4632,7 @@
       {
         "failWrite": {
           "error": {
-            "code": 4
+            "code": 2
           },
           "expectUserCallback": false
         }
@@ -4753,9 +4696,9 @@
       }
     ]
   },
-  "Writes that fail with code resource-exhausted are retried": {
+  "Writes that fail with code deadline-exceeded are retried": {
     "describeName": "Writes:",
-    "itName": "Writes that fail with code resource-exhausted are retried",
+    "itName": "Writes that fail with code deadline-exceeded are retried",
     "tags": [],
     "config": {
       "useGarbageCollection": true
@@ -4816,7 +4759,7 @@
       {
         "failWrite": {
           "error": {
-            "code": 8
+            "code": 4
           },
           "expectUserCallback": false
         }