Implement retrieving data from remote storage

Related #604

lib/src/main/java/org/asamk/signal/manager/Manager.java

@@ -42,6 +42,7 @@ import org.asamk.signal.manager.helper.IncomingMessageHandler;
 import org.asamk.signal.manager.helper.PinHelper;
 import org.asamk.signal.manager.helper.ProfileHelper;
 import org.asamk.signal.manager.helper.SendHelper;
+import org.asamk.signal.manager.helper.StorageHelper;
 import org.asamk.signal.manager.helper.SyncHelper;
 import org.asamk.signal.manager.helper.UnidentifiedAccessHelper;
 import org.asamk.signal.manager.jobs.Context;
@@ -133,6 +134,7 @@ public class Manager implements Closeable {
 
     private final ProfileHelper profileHelper;
     private final PinHelper pinHelper;
+    private final StorageHelper storageHelper;
     private final SendHelper sendHelper;
     private final SyncHelper syncHelper;
     private final AttachmentHelper attachmentHelper;
@@ -209,6 +211,7 @@ public class Manager implements Closeable {
                 avatarStore,
                 this::resolveSignalServiceAddress,
                 account.getRecipientStore());
+        this.storageHelper = new StorageHelper(account, dependencies, groupHelper);
         this.contactHelper = new ContactHelper(account);
         this.syncHelper = new SyncHelper(account,
                 attachmentHelper,
@@ -223,7 +226,8 @@ public class Manager implements Closeable {
                 sendHelper,
                 groupHelper,
                 syncHelper,
-                profileHelper);
+                profileHelper,
+                storageHelper);
         var jobExecutor = new JobExecutor(context);
 
         this.incomingMessageHandler = new IncomingMessageHandler(account,
@@ -747,6 +751,13 @@ public class Manager implements Closeable {
 
     public void requestAllSyncData() throws IOException {
         syncHelper.requestAllSyncData();
+        retrieveRemoteStorage();
+    }
+
+    void retrieveRemoteStorage() throws IOException {
+        if (account.getStorageKey() != null) {
+            storageHelper.readDataFromStorage();
+        }
     }
 
     private byte[] getSenderCertificate() {

lib/src/main/java/org/asamk/signal/manager/ProvisioningManager.java

@@ -103,6 +103,7 @@ public class ProvisioningManager {
                 ? null
                 : DeviceNameUtil.encryptDeviceName(deviceName, ret.getIdentity().getPrivateKey());
 
+        logger.debug("Finishing new device registration");
         var deviceId = accountManager.finishNewDeviceRegistration(ret.getProvisioningCode(),
                 false,
                 true,
@@ -129,6 +130,7 @@ public class ProvisioningManager {
             try {
                 m = new Manager(account, pathConfig, serviceEnvironmentConfig, userAgent);
 
+                logger.debug("Refreshing pre keys");
                 try {
                     m.refreshPreKeys();
                 } catch (Exception e) {
@@ -136,6 +138,7 @@ public class ProvisioningManager {
                     throw e;
                 }
 
+                logger.debug("Requesting sync data");
                 try {
                     m.requestAllSyncData();
                 } catch (Exception e) {

lib/src/main/java/org/asamk/signal/manager/RegistrationManager.java

@@ -174,7 +174,6 @@ public class RegistrationManager implements Closeable {
             masterKey = registrationLockData.getMasterKey();
         }
 
-        // TODO response.isStorageCapable()
         //accountManager.setGcmId(Optional.of(GoogleCloudMessaging.getInstance(this).register(REGISTRATION_ID)));
         account.finishRegistration(UuidUtil.parseOrNull(response.getUuid()), masterKey, pin);
 
@@ -186,6 +185,9 @@ public class RegistrationManager implements Closeable {
             m.refreshPreKeys();
             // Set an initial empty profile so user can be added to groups
             m.setProfile(null, null, null, null, null);
+            if (response.isStorageCapable()) {
+                m.retrieveRemoteStorage();
+            }
 
             final var result = m;
             m = null;

lib/src/main/java/org/asamk/signal/manager/TrustLevel.java

@@ -1,6 +1,7 @@
 package org.asamk.signal.manager;
 
 import org.whispersystems.signalservice.api.messages.multidevice.VerifiedMessage;
+import org.whispersystems.signalservice.internal.storage.protos.ContactRecord;
 
 public enum TrustLevel {
     UNTRUSTED,
@@ -16,6 +17,20 @@ public enum TrustLevel {
         return TrustLevel.cachedValues[i];
     }
 
+    public static TrustLevel fromIdentityState(ContactRecord.IdentityState identityState) {
+        switch (identityState) {
+            case DEFAULT:
+                return TRUSTED_UNVERIFIED;
+            case UNVERIFIED:
+                return UNTRUSTED;
+            case VERIFIED:
+                return TRUSTED_VERIFIED;
+            case UNRECOGNIZED:
+                return null;
+        }
+        throw new RuntimeException("Unknown identity state: " + identityState);
+    }
+
     public static TrustLevel fromVerifiedState(VerifiedMessage.VerifiedState verifiedState) {
         switch (verifiedState) {
             case DEFAULT:

lib/src/main/java/org/asamk/signal/manager/actions/RetrieveStorageDataAction.java

@@ -0,0 +1,26 @@
+package org.asamk.signal.manager.actions;
+
+import org.asamk.signal.manager.jobs.Context;
+
+public class RetrieveStorageDataAction implements HandleAction {
+
+    private static final RetrieveStorageDataAction INSTANCE = new RetrieveStorageDataAction();
+
+    private RetrieveStorageDataAction() {
+    }
+
+    public static RetrieveStorageDataAction create() {
+        return INSTANCE;
+    }
+
+    @Override
+    public void execute(Context context) throws Throwable {
+        if (context.getAccount().getStorageKey() != null) {
+            context.getStorageHelper().readDataFromStorage();
+        } else {
+            if (!context.getAccount().isMasterDevice()) {
+                context.getSyncHelper().requestAllSyncData();
+            }
+        }
+    }
+}

lib/src/main/java/org/asamk/signal/manager/actions/SendSyncKeysAction.java

@@ -0,0 +1,20 @@
+package org.asamk.signal.manager.actions;
+
+import org.asamk.signal.manager.jobs.Context;
+
+public class SendSyncKeysAction implements HandleAction {
+
+    private static final SendSyncKeysAction INSTANCE = new SendSyncKeysAction();
+
+    private SendSyncKeysAction() {
+    }
+
+    public static SendSyncKeysAction create() {
+        return INSTANCE;
+    }
+
+    @Override
+    public void execute(Context context) throws Throwable {
+        context.getSyncHelper().sendKeysMessage();
+    }
+}

lib/src/main/java/org/asamk/signal/manager/helper/IncomingMessageHandler.java

@@ -8,12 +8,14 @@ import org.asamk.signal.manager.UntrustedIdentityException;
 import org.asamk.signal.manager.actions.HandleAction;
 import org.asamk.signal.manager.actions.RenewSessionAction;
 import org.asamk.signal.manager.actions.RetrieveProfileAction;
+import org.asamk.signal.manager.actions.RetrieveStorageDataAction;
 import org.asamk.signal.manager.actions.SendGroupInfoAction;
 import org.asamk.signal.manager.actions.SendGroupInfoRequestAction;
 import org.asamk.signal.manager.actions.SendReceiptAction;
 import org.asamk.signal.manager.actions.SendSyncBlockedListAction;
 import org.asamk.signal.manager.actions.SendSyncContactsAction;
 import org.asamk.signal.manager.actions.SendSyncGroupsAction;
+import org.asamk.signal.manager.actions.SendSyncKeysAction;
 import org.asamk.signal.manager.groups.GroupId;
 import org.asamk.signal.manager.groups.GroupNotFoundException;
 import org.asamk.signal.manager.groups.GroupUtils;
@@ -237,7 +239,10 @@ public final class IncomingMessageHandler {
             if (rm.isBlockedListRequest()) {
                 actions.add(SendSyncBlockedListAction.create());
             }
-            // TODO Handle rm.isConfigurationRequest(); rm.isKeysRequest();
+            if (rm.isKeysRequest()) {
+                actions.add(SendSyncKeysAction.create());
+            }
+            // TODO Handle rm.isConfigurationRequest();
         }
         if (syncMessage.getGroups().isPresent()) {
             logger.warn("Received a group v1 sync message, that can't be handled anymore, ignoring.");
@@ -307,7 +312,7 @@ public final class IncomingMessageHandler {
                 case LOCAL_PROFILE:
                     actions.add(new RetrieveProfileAction(account.getSelfRecipientId()));
                 case STORAGE_MANIFEST:
-                    // TODO
+                    actions.add(RetrieveStorageDataAction.create());
             }
         }
         if (syncMessage.getKeys().isPresent()) {
@@ -315,6 +320,7 @@ public final class IncomingMessageHandler {
             if (keysMessage.getStorageService().isPresent()) {
                 final var storageKey = keysMessage.getStorageService().get();
                 account.setStorageKey(storageKey);
+                actions.add(RetrieveStorageDataAction.create());
             }
         }
         if (syncMessage.getConfiguration().isPresent()) {

lib/src/main/java/org/asamk/signal/manager/helper/StorageHelper.java

@@ -0,0 +1,222 @@
+package org.asamk.signal.manager.helper;
+
+import org.asamk.signal.manager.SignalDependencies;
+import org.asamk.signal.manager.TrustLevel;
+import org.asamk.signal.manager.groups.GroupId;
+import org.asamk.signal.manager.storage.SignalAccount;
+import org.asamk.signal.manager.storage.recipients.Contact;
+import org.signal.zkgroup.InvalidInputException;
+import org.signal.zkgroup.groups.GroupMasterKey;
+import org.signal.zkgroup.profiles.ProfileKey;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.whispersystems.libsignal.IdentityKey;
+import org.whispersystems.libsignal.InvalidKeyException;
+import org.whispersystems.libsignal.util.guava.Optional;
+import org.whispersystems.signalservice.api.storage.SignalAccountRecord;
+import org.whispersystems.signalservice.api.storage.SignalStorageManifest;
+import org.whispersystems.signalservice.api.storage.SignalStorageRecord;
+import org.whispersystems.signalservice.api.storage.StorageId;
+import org.whispersystems.signalservice.internal.storage.protos.ManifestRecord;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class StorageHelper {
+
+    private final static Logger logger = LoggerFactory.getLogger(StorageHelper.class);
+
+    private final SignalAccount account;
+    private final SignalDependencies dependencies;
+    private final GroupHelper groupHelper;
+
+    public StorageHelper(
+            final SignalAccount account, final SignalDependencies dependencies, final GroupHelper groupHelper
+    ) {
+        this.account = account;
+        this.dependencies = dependencies;
+        this.groupHelper = groupHelper;
+    }
+
+    public void readDataFromStorage() throws IOException {
+        logger.debug("Reading data from remote storage");
+        Optional<SignalStorageManifest> manifest;
+        try {
+            manifest = dependencies.getAccountManager()
+                    .getStorageManifestIfDifferentVersion(account.getStorageKey(), account.getStorageManifestVersion());
+        } catch (InvalidKeyException e) {
+            logger.warn("Manifest couldn't be decrypted, ignoring.");
+            return;
+        }
+
+        if (!manifest.isPresent()) {
+            logger.debug("Manifest is up to date, does not exist or couldn't be decrypted, ignoring.");
+            return;
+        }
+
+        account.setStorageManifestVersion(manifest.get().getVersion());
+
+        readAccountRecord(manifest.get());
+
+        final var storageIds = manifest.get()
+                .getStorageIds()
+                .stream()
+                .filter(id -> !id.isUnknown() && id.getType() != ManifestRecord.Identifier.Type.ACCOUNT_VALUE)
+                .collect(Collectors.toList());
+
+        for (final var record : getSignalStorageRecords(storageIds)) {
+            if (record.getType() == ManifestRecord.Identifier.Type.GROUPV2_VALUE) {
+                readGroupV2Record(record);
+            } else if (record.getType() == ManifestRecord.Identifier.Type.GROUPV1_VALUE) {
+                readGroupV1Record(record);
+            } else if (record.getType() == ManifestRecord.Identifier.Type.CONTACT_VALUE) {
+                readContactRecord(record);
+            }
+        }
+    }
+
+    private void readContactRecord(final SignalStorageRecord record) {
+        if (record == null || !record.getContact().isPresent()) {
+            return;
+        }
+
+        final var contactRecord = record.getContact().get();
+        final var address = contactRecord.getAddress();
+
+        final var recipientId = account.getRecipientStore().resolveRecipient(address);
+        final var contact = account.getContactStore().getContact(recipientId);
+        if (contactRecord.getGivenName().isPresent() || contactRecord.getFamilyName().isPresent() || (
+                (contact == null || !contact.isBlocked()) && contactRecord.isBlocked()
+        )) {
+            final var newContact = (contact == null ? Contact.newBuilder() : Contact.newBuilder(contact)).withBlocked(
+                    contactRecord.isBlocked())
+                    .withName((contactRecord.getGivenName().or("") + " " + contactRecord.getFamilyName().or("")).trim())
+                    .build();
+            account.getContactStore().storeContact(recipientId, newContact);
+        }
+
+        if (contactRecord.getProfileKey().isPresent()) {
+            try {
+                final var profileKey = new ProfileKey(contactRecord.getProfileKey().get());
+                account.getProfileStore().storeProfileKey(recipientId, profileKey);
+            } catch (InvalidInputException e) {
+                logger.warn("Received invalid contact profile key from storage");
+            }
+        }
+        if (contactRecord.getIdentityKey().isPresent()) {
+            try {
+                final var identityKey = new IdentityKey(contactRecord.getIdentityKey().get());
+                account.getIdentityKeyStore().saveIdentity(recipientId, identityKey, new Date());
+
+                final var trustLevel = TrustLevel.fromIdentityState(contactRecord.getIdentityState());
+                if (trustLevel != null) {
+                    account.getIdentityKeyStore().setIdentityTrustLevel(recipientId, identityKey, trustLevel);
+                }
+            } catch (InvalidKeyException e) {
+                logger.warn("Received invalid contact identity key from storage");
+            }
+        }
+    }
+
+    private void readGroupV1Record(final SignalStorageRecord record) {
+        if (record == null || !record.getGroupV1().isPresent()) {
+            return;
+        }
+
+        final var groupV1Record = record.getGroupV1().get();
+        final var groupIdV1 = GroupId.v1(groupV1Record.getGroupId());
+
+        final var group = account.getGroupStore().getGroup(groupIdV1);
+        if (group == null) {
+            try {
+                groupHelper.sendGroupInfoRequest(groupIdV1, account.getSelfRecipientId());
+            } catch (Throwable e) {
+                logger.warn("Failed to send group request", e);
+            }
+        }
+        final var groupV1 = account.getGroupStore().getOrCreateGroupV1(groupIdV1);
+        if (groupV1.isBlocked() != groupV1Record.isBlocked()) {
+            groupV1.setBlocked(groupV1Record.isBlocked());
+            account.getGroupStore().updateGroup(groupV1);
+        }
+    }
+
+    private void readGroupV2Record(final SignalStorageRecord record) {
+        if (record == null || !record.getGroupV2().isPresent()) {
+            return;
+        }
+
+        final var groupV2Record = record.getGroupV2().get();
+        if (groupV2Record.isArchived()) {
+            return;
+        }
+
+        final GroupMasterKey groupMasterKey;
+        try {
+            groupMasterKey = new GroupMasterKey(groupV2Record.getMasterKeyBytes());
+        } catch (InvalidInputException e) {
+            logger.warn("Received invalid group master key from storage");
+            return;
+        }
+
+        final var group = groupHelper.getOrMigrateGroup(groupMasterKey, 0, null);
+        if (group.isBlocked() != groupV2Record.isBlocked()) {
+            group.setBlocked(groupV2Record.isBlocked());
+            account.getGroupStore().updateGroup(group);
+        }
+    }
+
+    private void readAccountRecord(final SignalStorageManifest manifest) throws IOException {
+        Optional<StorageId> accountId = manifest.getAccountStorageId();
+        if (!accountId.isPresent()) {
+            logger.warn("Manifest has no account record, ignoring.");
+            return;
+        }
+
+        SignalStorageRecord record = getSignalStorageRecord(accountId.get());
+        if (record == null) {
+            logger.warn("Could not find account record, even though we had an ID, ignoring.");
+            return;
+        }
+
+        SignalAccountRecord accountRecord = record.getAccount().orNull();
+        if (accountRecord == null) {
+            logger.warn("The storage record didn't actually have an account, ignoring.");
+            return;
+        }
+
+        if (accountRecord.getProfileKey().isPresent()) {
+            try {
+                account.setProfileKey(new ProfileKey(accountRecord.getProfileKey().get()));
+            } catch (InvalidInputException e) {
+                logger.warn("Received invalid profile key from storage");
+            }
+        }
+    }
+
+    private SignalStorageRecord getSignalStorageRecord(final StorageId accountId) throws IOException {
+        List<SignalStorageRecord> records;
+        try {
+            records = dependencies.getAccountManager()
+                    .readStorageRecords(account.getStorageKey(), Collections.singletonList(accountId));
+        } catch (InvalidKeyException e) {
+            logger.warn("Failed to read storage records, ignoring.");
+            return null;
+        }
+        return records.size() > 0 ? records.get(0) : null;
+    }
+
+    private List<SignalStorageRecord> getSignalStorageRecords(final List<StorageId> storageIds) throws IOException {
+        List<SignalStorageRecord> records;
+        try {
+            records = dependencies.getAccountManager().readStorageRecords(account.getStorageKey(), storageIds);
+        } catch (InvalidKeyException e) {
+            logger.warn("Failed to read storage records, ignoring.");
+            return List.of();
+        }
+        return records;
+    }
+}

lib/src/main/java/org/asamk/signal/manager/helper/SyncHelper.java

@@ -20,6 +20,7 @@ import org.whispersystems.signalservice.api.messages.multidevice.DeviceContactsI
 import org.whispersystems.signalservice.api.messages.multidevice.DeviceContactsOutputStream;
 import org.whispersystems.signalservice.api.messages.multidevice.DeviceGroup;
 import org.whispersystems.signalservice.api.messages.multidevice.DeviceGroupsOutputStream;
+import org.whispersystems.signalservice.api.messages.multidevice.KeysMessage;
 import org.whispersystems.signalservice.api.messages.multidevice.RequestMessage;
 import org.whispersystems.signalservice.api.messages.multidevice.SignalServiceSyncMessage;
 import org.whispersystems.signalservice.api.messages.multidevice.VerifiedMessage;
@@ -215,6 +216,11 @@ public class SyncHelper {
         sendHelper.sendSyncMessage(SignalServiceSyncMessage.forVerified(verifiedMessage));
     }
 
+    public void sendKeysMessage() throws IOException {
+        var keysMessage = new KeysMessage(Optional.fromNullable(account.getStorageKey()));
+        sendHelper.sendSyncMessage(SignalServiceSyncMessage.forKeys(keysMessage));
+    }
+
     public void handleSyncDeviceContacts(final InputStream input) throws IOException {
         final var s = new DeviceContactsInputStream(input);
         DeviceContact c;

lib/src/main/java/org/asamk/signal/manager/jobs/Context.java

@@ -5,6 +5,7 @@ import org.asamk.signal.manager.StickerPackStore;
 import org.asamk.signal.manager.helper.GroupHelper;
 import org.asamk.signal.manager.helper.ProfileHelper;
 import org.asamk.signal.manager.helper.SendHelper;
+import org.asamk.signal.manager.helper.StorageHelper;
 import org.asamk.signal.manager.helper.SyncHelper;
 import org.asamk.signal.manager.storage.SignalAccount;
 
@@ -17,6 +18,7 @@ public class Context {
     private final GroupHelper groupHelper;
     private final SyncHelper syncHelper;
     private final ProfileHelper profileHelper;
+    private final StorageHelper storageHelper;
 
     public Context(
             final SignalAccount account,
@@ -25,7 +27,8 @@ public class Context {
             final SendHelper sendHelper,
             final GroupHelper groupHelper,
             final SyncHelper syncHelper,
-            final ProfileHelper profileHelper
+            final ProfileHelper profileHelper,
+            final StorageHelper storageHelper
     ) {
         this.account = account;
         this.dependencies = dependencies;
@@ -34,6 +37,7 @@ public class Context {
         this.groupHelper = groupHelper;
         this.syncHelper = syncHelper;
         this.profileHelper = profileHelper;
+        this.storageHelper = storageHelper;
     }
 
     public SignalAccount getAccount() {
@@ -63,4 +67,8 @@ public class Context {
     public ProfileHelper getProfileHelper() {
         return profileHelper;
     }
+
+    public StorageHelper getStorageHelper() {
+        return storageHelper;
+    }
 }

lib/src/main/java/org/asamk/signal/manager/storage/SignalAccount.java

@@ -84,6 +84,7 @@ public class SignalAccount implements Closeable {
     private String registrationLockPin;
     private MasterKey pinMasterKey;
     private StorageKey storageKey;
+    private long storageManifestVersion = -1;
     private ProfileKey profileKey;
     private int preKeyIdOffset;
     private int nextSignedPreKeyId;
@@ -291,6 +292,9 @@ public class SignalAccount implements Closeable {
         this.registered = true;
         this.isMultiDevice = true;
         this.lastReceiveTimestamp = 0;
+        this.pinMasterKey = null;
+        this.storageManifestVersion = -1;
+        this.storageKey = null;
     }
 
     private void migrateLegacyConfigs() {
@@ -432,6 +436,9 @@ public class SignalAccount implements Closeable {
         if (rootNode.hasNonNull("storageKey")) {
             storageKey = new StorageKey(Base64.getDecoder().decode(rootNode.get("storageKey").asText()));
         }
+        if (rootNode.hasNonNull("storageManifestVersion")) {
+            storageManifestVersion = rootNode.get("storageManifestVersion").asLong();
+        }
         if (rootNode.hasNonNull("preKeyIdOffset")) {
             preKeyIdOffset = rootNode.get("preKeyIdOffset").asInt(0);
         } else {
@@ -693,6 +700,7 @@ public class SignalAccount implements Closeable {
                             pinMasterKey == null ? null : Base64.getEncoder().encodeToString(pinMasterKey.serialize()))
                     .put("storageKey",
                             storageKey == null ? null : Base64.getEncoder().encodeToString(storageKey.serialize()))
+                    .put("storageManifestVersion", storageManifestVersion == -1 ? null : storageManifestVersion)
                     .put("preKeyIdOffset", preKeyIdOffset)
                     .put("nextSignedPreKeyId", nextSignedPreKeyId)
                     .put("profileKey",
@@ -877,6 +885,18 @@ public class SignalAccount implements Closeable {
         save();
     }
 
+    public long getStorageManifestVersion() {
+        return this.storageManifestVersion;
+    }
+
+    public void setStorageManifestVersion(final long storageManifestVersion) {
+        if (storageManifestVersion == this.storageManifestVersion) {
+            return;
+        }
+        this.storageManifestVersion = storageManifestVersion;
+        save();
+    }
+
     public ProfileKey getProfileKey() {
         return profileKey;
     }
@@ -948,6 +968,8 @@ public class SignalAccount implements Closeable {
 
     public void finishRegistration(final UUID uuid, final MasterKey masterKey, final String pin) {
         this.pinMasterKey = masterKey;
+        this.storageManifestVersion = -1;
+        this.storageKey = null;
         this.encryptedDeviceName = null;
         this.deviceId = SignalServiceAddress.DEFAULT_DEVICE_ID;
         this.isMultiDevice = false;