(fix) improved EAS speed during initial syncing of large mailboxes (#3293)

ActiveSync/SOGoActiveSyncDispatcher+Sync.m

@@ -133,6 +133,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   [[o properties] removeObjectForKey: @"MoreAvailable"];
   [[o properties] removeObjectForKey: @"BodyPreferenceType"];
   [[o properties] removeObjectForKey: @"SuccessfulMoveItemsOps"];
+  [[o properties] removeObjectForKey: @"InitialLoadSequence"];
 
   [[o properties] addEntriesFromDictionary: values];
   [o save];
@@ -673,7 +674,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         NSDictionary *component;
         NSArray *allComponents;
 
-        BOOL updated;
+        BOOL updated, initialLoadInProgress;
         int deleted, return_count;
           
         if (theFolderType == ActiveSyncContactFolder)
@@ -683,7 +684,20 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         else
           component_name = @"vtodo";
 
-        allComponents = [theCollection syncTokenFieldsWithProperties: nil   matchingSyncToken: theSyncKey  fromDate: theFilterType];
+        initialLoadInProgress = NO;
+
+        if ([theSyncKey isEqualToString: @"-1"])
+            [folderMetadata setObject: davCollectionTagToStore forKey: @"InitialLoadSequence"];
+
+        if ([folderMetadata objectForKey: @"InitialLoadSequence"])
+          {
+            if ([theSyncKey intValue] < [[folderMetadata objectForKey: @"InitialLoadSequence"] intValue])
+              initialLoadInProgress = YES;
+            else
+              [folderMetadata removeObjectForKey: @"InitialLoadSequence"];
+          }
+
+        allComponents = [theCollection syncTokenFieldsWithProperties: nil   matchingSyncToken: theSyncKey  fromDate: theFilterType initialLoad: initialLoadInProgress];
         allComponents = [allComponents sortedArrayUsingDescriptors: [NSArray arrayWithObjects: [[NSSortDescriptor alloc] initWithKey: @"c_lastmodified" ascending:YES], nil]];
 
         
@@ -847,12 +861,35 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         NSMutableArray *allCacheObjects, *sortedBySequence;
 
         SOGoMailObject *mailObject;
-        NSArray *allMessages;
+        NSArray *allMessages, *a;
 
-        int j, k, return_count;
+        int j, k, return_count, highestmodseq;
-        BOOL found_in_cache;
+        BOOL found_in_cache, initialLoadInProgress;
 
-        allMessages = [theCollection syncTokenFieldsWithProperties: nil   matchingSyncToken: theSyncKey  fromDate: theFilterType];
+        initialLoadInProgress = NO; 
+
+        if ([theSyncKey isEqualToString: @"-1"])
+          {
+            highestmodseq = 0;
+
+            a = [[theCollection davCollectionTag] componentsSeparatedByString: @"-"];
+            [folderMetadata setObject: [a objectAtIndex: 1] forKey: @"InitialLoadSequence"];
+          }
+        else
+         {
+           a = [theSyncKey componentsSeparatedByString: @"-"];
+           highestmodseq = [[a objectAtIndex: 1] intValue];
+         }
+
+        if ([folderMetadata objectForKey: @"InitialLoadSequence"])
+          {
+            if (highestmodseq < [[folderMetadata objectForKey: @"InitialLoadSequence"] intValue])
+              initialLoadInProgress = YES;
+            else
+              [folderMetadata removeObjectForKey: @"InitialLoadSequence"];
+          }
+
+        allMessages = [theCollection syncTokenFieldsWithProperties: nil  matchingSyncToken: theSyncKey  fromDate: theFilterType  initialLoad: initialLoadInProgress];
         max = [allMessages count];
         
         allCacheObjects = [NSMutableArray array];
@@ -911,7 +948,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
                 NSString *lastSequence;
                 more_available = YES;
                 
-                lastSequence = ([[aCacheObject sequence] isEqual: [NSNull null]] ? @"1" : [aCacheObject sequence]);
+                lastSequence = ([[aCacheObject sequence] isEqual: [NSNull null]] ? [NSString stringWithFormat:@"%d", highestmodseq] : [aCacheObject sequence]);
                 *theLastServerKey = [[NSString alloc] initWithFormat: @"%@-%@", [aCacheObject uid], lastSequence];
                 //NSLog(@"Reached windowSize - lastUID will be: %@", *theLastServerKey);
                 DESTROY(pool);

ActiveSync/SOGoActiveSyncDispatcher.m

@@ -944,6 +944,7 @@ static BOOL debugOn = NO;
          [[o properties] removeObjectForKey: @"MoreAvailable"];
          [[o properties] removeObjectForKey: @"BodyPreferenceType"];
          [[o properties] removeObjectForKey: @"SuccessfulMoveItemsOps"];
+         [[o properties] removeObjectForKey: @"InitialLoadSequence"];
          [o save];
               
          command_count++;
@@ -1029,6 +1030,7 @@ static BOOL debugOn = NO;
                    [[o properties] removeObjectForKey: @"MoreAvailable"];
                    [[o properties] removeObjectForKey: @"BodyPreferenceType"];
                    [[o properties] removeObjectForKey: @"SuccessfulMoveItemsOps"];
+                   [[o properties] removeObjectForKey: @"InitialLoadSequence"];
                  }
 
                [o save];
@@ -1052,6 +1054,7 @@ static BOOL debugOn = NO;
                    [[o properties] removeObjectForKey: @"MoreAvailable"];
                    [[o properties] removeObjectForKey: @"BodyPreferenceType"];
                    [[o properties] removeObjectForKey: @"SuccessfulMoveItemsOps"];
+                   [[o properties] removeObjectForKey: @"InitialLoadSequence"];
                  }
 
                [o save];
@@ -1210,7 +1213,7 @@ static BOOL debugOn = NO;
            filter = [NSCalendarDate dateFromFilterType: [[(id)[[allCollections objectAtIndex: j] getElementsByTagName: @"FilterType"] lastObject] textValue]];
            syncKey = [[(id)[[allCollections objectAtIndex: j] getElementsByTagName: @"SyncKey"] lastObject] textValue];
       
-           allMessages = [currentCollection syncTokenFieldsWithProperties: nil  matchingSyncToken: syncKey  fromDate: filter];
+           allMessages = [currentCollection syncTokenFieldsWithProperties: nil  matchingSyncToken: syncKey  fromDate: filter initialLoad: NO];
 
            count = [allMessages count];
       

NEWS

@@ -3,6 +3,7 @@
 
 Enhancements
  - improved EAS speed and memory usage, avoiding many IMAP LIST commands (#3294)
+ - improved EAS speed during initial syncing of large mailboxes (#3293)
 
 Bug fixes
  - fixed display of whitelisted attendees in Preferences window on Firefox (#3285)

SoObjects/Mailer/SOGoMailFolder.h

@@ -98,7 +98,8 @@
 
 - (NSArray *) syncTokenFieldsWithProperties: (NSDictionary *) properties
                           matchingSyncToken: (NSString *) syncToken
-                                   fromDate: (NSCalendarDate *) theStartDate;
+                                   fromDate: (NSCalendarDate *) theStartDate
+                                initialLoad: (BOOL) initialLoadInProgress;
 
 /* flags */
 

SoObjects/Mailer/SOGoMailFolder.m

@@ -2090,6 +2090,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data)
 - (NSArray *) syncTokenFieldsWithProperties: (NSArray *) theProperties
                           matchingSyncToken: (NSString *) theSyncToken
                                    fromDate: (NSCalendarDate *) theStartDate
+                                initialLoad: (BOOL) initialLoadInProgress
 {
   EOQualifier *searchQualifier;
   NSMutableArray *allTokens;
@@ -2166,6 +2167,9 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data)
 
   for (i = 0; i < [fetchResults count]; i++)
     { 
+      if ([[[fetchResults objectAtIndex: i] objectForKey: @"flags"] containsObject: @"deleted"] && initialLoadInProgress)
+        continue;
+
       d = [NSDictionary dictionaryWithObject: ([[[fetchResults objectAtIndex: i] objectForKey: @"flags"] containsObject: @"deleted"]) ? [NSNull null] : [[fetchResults objectAtIndex: i] objectForKey: @"modseq"]
                                       forKey: [[[fetchResults objectAtIndex: i] objectForKey: @"uid"] stringValue]];
       [allTokens addObject: d];
@@ -2176,7 +2180,7 @@ _compareFetchResultsByMODSEQ (id entry1, id entry2, void *data)
   if (highestmodseq == 0)
     highestmodseq = 1;
 
-  if (highestmodseq > 0)
+  if (highestmodseq > 0 && !initialLoadInProgress)
     {
       id uid;
 

SoObjects/SOGo/SOGoGCSFolder.h

@@ -128,8 +128,8 @@
 
 - (NSArray *) syncTokenFieldsWithProperties: (NSDictionary *) properties
                           matchingSyncToken: (NSString *) syncToken
-                                   fromDate: (NSCalendarDate *) theStartDate;
+                                   fromDate: (NSCalendarDate *) theStartDate
-
+                                initialLoad: (BOOL) initialLoadInProgress;
 
 /* multiget helper */
 - (WOResponse *) performMultigetInContext: (WOContext *) queryContext

SoObjects/SOGo/SOGoGCSFolder.m

@@ -1151,6 +1151,7 @@ static NSArray *childRecordFields = nil;
 - (NSArray *) syncTokenFieldsWithProperties: (NSDictionary *) properties
                           matchingSyncToken: (NSString *) syncToken
                                    fromDate: (NSCalendarDate *) theStartDate
+                                initialLoad: (BOOL) initialLoadInProgress
 {
   /* TODO:
      - validation:
@@ -1200,6 +1201,8 @@ static NSArray *childRecordFields = nil;
       mRecords = [NSMutableArray arrayWithArray: [self _fetchFields: fields
                                                       withQualifier: qualifier
                                                       ignoreDeleted: YES]];
+      if (initialLoadInProgress)
+        {
           qualifier = [EOQualifier qualifierWithQualifierFormat:
                                    @"c_lastmodified > %d and c_deleted == 1",
                                    syncTokenInt];
@@ -1207,6 +1210,8 @@ static NSArray *childRecordFields = nil;
           [mRecords addObjectsFromArray: [self _fetchFields: fields
                                               withQualifier: qualifier
                                               ignoreDeleted: NO]];
+        }
+
       records = mRecords;
     }
   else
@@ -1499,7 +1504,9 @@ static NSArray *childRecordFields = nil;
       properties = [self parseDAVRequestedProperties: propElement];
       records = [self syncTokenFieldsWithProperties: properties
                                   matchingSyncToken: syncToken
-                                           fromDate: nil];
+                                           fromDate: nil
+                                        initialLoad: NO];
+
       [self _appendComponentProperties: [properties allKeys]
                            fromRecords: records
                      matchingSyncToken: [syncToken intValue]