Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Centralize/Unify Sync and Submit Logic #628

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Centralize/Unify Sync and Submit Logic #628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1e5f78
Refactor sync towards being unifiable between platforms, and add test…
ctsims Aug 11, 2017
947d88e
Merge branch 'ctsims/b_64_lib_move' into ctsims/async_collison_replic…
ctsims Aug 11, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
186 changes: 118 additions & 68 deletions src/cli/java/org/commcare/util/cli/ApplicationHost.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,19 @@
import org.commcare.core.parse.ParseUtils;
import org.commcare.core.sandbox.SandboxUtils;
import org.commcare.data.xml.DataModelPullParser;
import org.commcare.modern.session.SessionWrapper;
import org.commcare.session.SessionFrame;
import org.commcare.suite.model.FormIdDatum;
import org.commcare.suite.model.SessionDatum;
import org.commcare.suite.model.StackFrameStep;
import org.commcare.util.CommCarePlatform;
import org.commcare.util.engine.CommCareConfigEngine;
import org.commcare.util.mocks.CLISessionWrapper;
import org.commcare.util.mocks.CoreNetworkContext;
import org.commcare.util.mocks.JavaPlatformFormSubmitTool;
import org.commcare.util.mocks.JavaPlatformSyncTool;
import org.commcare.util.mocks.MockUserDataSandbox;
import org.commcare.util.mocks.SyncStateMachine;
import org.commcare.util.screen.CommCareSessionException;
import org.commcare.util.screen.EntityScreen;
import org.commcare.util.screen.MenuScreen;
Expand Down Expand Up @@ -50,6 +55,7 @@
import java.net.HttpURLConnection;
import java.net.PasswordAuthentication;
import java.net.URL;
import java.util.ArrayList;

/**
* CLI host for running a commcare application which has been configured and instatiated
Expand All @@ -75,6 +81,9 @@ public class ApplicationHost {
private String mRestoreFile;
private boolean mRestoreStrategySet = false;

private ArrayList<byte[]> formQueue = new ArrayList<>();
private boolean submissionsEnabled = false;

public ApplicationHost(CommCareConfigEngine engine, PrototypeFactory prototypeFactory) {
this.mEngine = engine;
this.mPlatform = engine.getPlatform();
Expand Down Expand Up @@ -201,6 +210,11 @@ private boolean loopSession() throws IOException {
continue;
}

if(input.equals(":submit")) {
attemptFormSubmissions();
continue;
}

if (input.startsWith(":lang")) {
String[] langArgs = input.split(" ");
if (langArgs.length != 2) {
Expand Down Expand Up @@ -257,6 +271,9 @@ private boolean loopSession() throws IOException {
if (!processResultInstance(player.getResultStream())) {
return true;
}
if(submissionsEnabled) {
formQueue.add(player.getResultBytes());
}
finishSession();
return true;
} else if (player.getExecutionResult() == XFormPlayer.FormResult.Cancelled) {
Expand All @@ -272,6 +289,33 @@ private boolean loopSession() throws IOException {
return true;
}

private void attemptFormSubmissions() {
if(!submissionsEnabled) {
System.out.println("Form submissions are not enabled! Please restart the cli with the 's' flag");
return;
}

if(this.formQueue.size() == 0) {
System.out.println("No pending forms");
return;
}
JavaPlatformFormSubmitTool submitter =
new JavaPlatformFormSubmitTool(this.mSandbox, new CoreNetworkContext(mLocalUserCredentials[0],mLocalUserCredentials[1]));

System.out.println(String.format("Submitting %d forms to %s", this.formQueue.size(), submitter.getSubmitUrl()));

ArrayList<byte[]> copyQueue = new ArrayList<>(formQueue);
for(int i = 0 ; i < copyQueue.size() ; ++i) {
byte[] form = copyQueue.get(i);
System.out.println(String.format("Submitting Form [%d/%d].....", i+1, this.formQueue.size()));
if(submitter.submitFormToServer(form)) {
formQueue.remove(i);
} else {
System.out.println("Ending form submission due to failure. Retry to continue");
}
}
}

private void printStack(CLISessionWrapper mSession) {
SessionFrame frame = mSession.getFrame();
System.out.println("Live Frame");
Expand Down Expand Up @@ -411,88 +455,82 @@ private void initUser() {
System.out.println("Setting logged in user to: " + u.getUsername());
}

public static void restoreUserToSandbox(UserSandbox sandbox, CLISessionWrapper session,
public static void restoreUserToSandbox(UserSandbox sandbox, SessionWrapper session,
String username, final String password) {
String urlStateParams = "";

boolean failed = true;

boolean incremental = false;

if (sandbox.getLoggedInUser() != null) {
String syncToken = sandbox.getSyncToken();
String caseStateHash = CaseDBUtils.computeCaseDbHash(sandbox.getCaseStorage());

urlStateParams = String.format("&since=%s&state=ccsh:%s", syncToken, caseStateHash);
incremental = true;

System.out.println(String.format(
"\nIncremental sync requested. \nSync Token: %s\nState Hash: %s",
syncToken, caseStateHash));
}

//fetch the restore data and set credentials
String otaFreshRestoreUrl = PropertyManager.instance().getSingularProperty("ota-restore-url") +
"?version=2.0";

String otaSyncUrl = otaFreshRestoreUrl + urlStateParams;

String domain = PropertyManager.instance().getSingularProperty("cc_user_domain");
final String qualifiedUsername = username + "@" + domain;

Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(qualifiedUsername, password.toCharArray());
}
});

//Go get our sandbox!
try {
System.out.println("GET: " + otaSyncUrl);
URL url = new URL(otaSyncUrl);
HttpURLConnection conn = (HttpURLConnection)url.openConnection();

if (conn.getResponseCode() == 412) {
System.out.println("Server Response 412 - The user sandbox is not consistent with " +
"the server's data. \n\nThis is expected if you have changed cases locally, " +
"since data is not sent to the server for updates. \n\nServer response cannot be restored," +
" you will need to restart the user's session to get new data.");
} else if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
System.out.println("\nInvalid username or password!");
} else if (conn.getResponseCode() >= 200 && conn.getResponseCode() < 300) {
JavaPlatformSyncTool tool = new JavaPlatformSyncTool(username, password, sandbox, session);

System.out.println("Restoring user " + username + " to domain " + domain);

ParseUtils.parseIntoSandbox(new BufferedInputStream(conn.getInputStream()), sandbox);
if(attemptSync(tool)) {

System.out.println("User data processed, new state token: " + sandbox.getSyncToken());
failed = false;
} else {
System.out.println("Unclear/Unexpected server response code: " + conn.getResponseCode());
}
} catch (InvalidStructureException | IOException
| XmlPullParserException | UnfullfilledRequirementsException e) {
e.printStackTrace();
}

if (failed) {
if (!incremental) {
System.exit(-1);
}
} else {
//Initialize our User
for (IStorageIterator<User> iterator = sandbox.getUserStorage().iterate(); iterator.hasMore(); ) {
User u = iterator.nextRecord();
if (username.equalsIgnoreCase(u.getUsername())) {
sandbox.setLoggedInUser(u);
}
}

if (session != null) {
// old session data is now no longer valid
session.clearVolitiles();
}
}
}

if (session != null) {
// old session data is now no longer valid
session.clearVolitiles();
private static boolean attemptSync(JavaPlatformSyncTool tool) {
try {
tool.initialize();

while (tool.getCurrentState() == SyncStateMachine.State.Ready_For_Request) {

tool.performRequest();

switch (tool.getCurrentState()) {
case Waiting_For_Progress:
tool.processWaitSignal();
break;
case Recovery_Requested:
tool.transitionToRecoveryStrategy();
break;
case Recoverable_Error:
tool.resetFromError();
break;
}
}

tool.processPayload();
return true;
} catch(SyncStateMachine.SyncErrorException e) {
String errorMessage = String.format("Sync Failed While [%s]\nError: ",
e.getStateBeforeError());

switch(e.getSyncError()){
case Invalid_Credentials:
errorMessage += "Invalid user credentials";
break;
case Unexpected_Server_Response_Code:
errorMessage += "Unexpected server response code: " + tool.getServerResponseCode();
break;
case Network_Error:
errorMessage += "Network problems, please try to sync again";
break;
case Invalid_Payload:
errorMessage += "Invalid Server Response";
break;
case Unhandled_Situation:
errorMessage += "Unhandled Behavior";
break;
}
if(e.getMessage() != null) {
errorMessage +="\nDetails: " + e.getMessage();
}
System.out.println(errorMessage);

if(tool.getStrategy() == SyncStateMachine.Strategy.Fresh) {
System.exit(-1);
}
return false;
}
}

Expand Down Expand Up @@ -531,6 +569,15 @@ private void syncAndReport() {
performCasePurge(mSandbox);

if (mLocalUserCredentials != null) {

if(submissionsEnabled && this.formQueue.size() > 0) {
attemptFormSubmissions();
if(this.formQueue.size() > 0) {
System.out.println("Cancelling sync until all forms are submitted.");
return;
}
}

System.out.println("Requesting sync...");

restoreUserToSandbox(mSandbox, mSession, mLocalUserCredentials[0], mLocalUserCredentials[1]);
Expand Down Expand Up @@ -563,4 +610,7 @@ public static void performCasePurge(UserSandbox sandbox) {
}
}

public void setSubmissionsEnabled() {
submissionsEnabled = true;
}
}
14 changes: 13 additions & 1 deletion src/cli/java/org/commcare/util/cli/CliPlayCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,15 @@ protected Options getOptions() {
.required(false)
.optionalArg(false)
.build();
options.addOption(restoreFile).addOption(demoUser);
Option submissionEnabled = Option.builder("s")
.argName("SUBMIT_ENABLED")
.desc("Enable Submitting forms to the server")
.longOpt("enable-submission")
.required(false)
.optionalArg(false)
.build();

options.addOption(restoreFile).addOption(demoUser).addOption(submissionEnabled);

return options;
}
Expand All @@ -59,6 +67,10 @@ public void handle() {
CommCareConfigEngine engine = configureApp(resourcePath, prototypeFactory);
ApplicationHost host = new ApplicationHost(engine, prototypeFactory);

if(cmd.hasOption("s")) {
host.setSubmissionsEnabled();
}

if (cmd.hasOption("r")) {
host.setRestoreToLocalFile(cmd.getOptionValue("r"));
} else if (cmd.hasOption("d")) {
Expand Down
50 changes: 50 additions & 0 deletions src/cli/java/org/commcare/util/mocks/CoreNetworkContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package org.commcare.util.mocks;

import org.commcare.util.Base64;
import org.javarosa.core.services.PropertyManager;

import java.net.Authenticator;
import java.net.HttpURLConnection;
import java.net.PasswordAuthentication;
import java.util.HashMap;

/**
* Created by ctsims on 8/11/2017.
*/

public class CoreNetworkContext {
String username;
String password;

public CoreNetworkContext(String username, String password) {
this.username = username;
this.password = password;
}

public String getFullyQualifiedUsername() {
String domain = PropertyManager.instance().getSingularProperty("cc_user_domain");

return username + "@" + domain;

}

public void addAuthProperty(HttpURLConnection connection, HashMap<String, String> properties) {
String encodedCredentials = Base64.encode(String.format("%s:%s", getFullyQualifiedUsername(), password).getBytes());
String basicAuth = "Basic " + encodedCredentials;
properties.put("Authorization",basicAuth);
}

public void configureProperties(HttpURLConnection connection, HashMap<String, String> properties) {
for(String key : properties.keySet()) {
connection.setRequestProperty(key, properties.get(key));
}
}

public String getPlainUsername() {
return username;
}

public String getPassword() {
return new String(password);
}
}
Loading