diff --git a/.gitignore b/.gitignore index 79ea050..fbef39d 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,40 @@ target ehthumbs.db Icon? Thumbs.db + + +# Built application files +*.apk +*.ap_ + +# Files for the Dalvik VM +*.dex + +# Java class files +*.class + +# Generated files +bin/ +gen/ + +# Gradle files +.gradle/ +build/ + +# Local configuration file (sdk path, etc) +local.properties + +# Proguard folder generated by Eclipse +proguard/ + +# Log Files +*.log + +# Android Studio Navigation editor temp files +.navigation/ +.idea +*.iml + +# Android Studio captures folder +captures/ + diff --git a/AndroidArduinoOpenCV/res/values/strings.xml b/AndroidArduinoOpenCV/res/values/strings.xml index 699c00a..b0d48b0 100644 --- a/AndroidArduinoOpenCV/res/values/strings.xml +++ b/AndroidArduinoOpenCV/res/values/strings.xml @@ -4,19 +4,20 @@ Roogle rover Roogle rover - - + + + Send - You are not connected to a device + You are not connected to a device. Bluetooth was not enabled. Leaving Bluetooth Chat. - connecting... - connected: - not connected + Connecting... + Connected: + Not connected. - scanning for devices... - select a device to connect + Scanning for devices... + Select a device to connect No devices have been paired No devices found Paired Devices @@ -26,4 +27,4 @@ Connect a device Make discoverable - \ No newline at end of file + diff --git a/GestureVoiceCommander/res/drawable-hdpi/ic_launcher.png b/GestureVoiceCommander/res/drawable-hdpi/ic_launcher.png index d62b72a..c609872 100644 Binary files a/GestureVoiceCommander/res/drawable-hdpi/ic_launcher.png and b/GestureVoiceCommander/res/drawable-hdpi/ic_launcher.png differ diff --git a/GestureVoiceCommander/src/com/androidmontreal/gesture/commander/RoverLexicon.java b/GestureVoiceCommander/src/com/androidmontreal/gesture/commander/RoverLexicon.java index ffcd4dc..f89c616 100644 --- a/GestureVoiceCommander/src/com/androidmontreal/gesture/commander/RoverLexicon.java +++ b/GestureVoiceCommander/src/com/androidmontreal/gesture/commander/RoverLexicon.java @@ -1,191 +1,192 @@ package com.androidmontreal.gesture.commander; import java.util.ArrayList; - +import java.util.Locale; public class RoverLexicon { - private int language; - private int timer = 5; - - /* - * Languages - */ - public static final int EN = 1000; - public static final int FR = 1001; - public static final String EN_CARRIER_PHRASE = "I will tell the robot: "; - public static final String FR_CARRIER_PHRASE = "Je vais dire au robo: "; - - - /* - * Commands, use the order of these constants for - * the precedence order of the commands - * (STOP has the highest precedence) - */ - public static final int STOP = 0; - public static final int EXPLORE = 1; - public static final int FORWARD = 2; - public static final int REVERSE = 3; - public static final int TURNRIGHT = 6; - public static final int TURNLEFT = 7; - public static final int ROTATERIGHT = 4; - public static final int ROTATELEFT = 5; - - private ArrayList en; - private ArrayList fr; - - private int mCommandToExecute; - - public String stop() { - return "S"; - } - - public String explore() { - return "E"; - } - - public String forward() { - return "1F2F"; - } - - public String reverse() { - return "1R2R"; - } - - public String turnRight() { - return "1F"; - } - - public String turnLeft() { - return "2F"; - } - - public String rotateRight() { - return "1F2R"; - } - - public String rotateLeft() { - return "1R2F"; - } - - public void defineLanguages() { - en = new ArrayList(); - en.add(STOP, "stop:wait:don't:no:damn"); - en.add(EXPLORE, "explore:try"); - en.add(FORWARD, "forward:ahead"); - en.add(REVERSE, "reverse:back"); - en.add(ROTATERIGHT, "rotate&right"); - en.add(ROTATELEFT, "rotate&left"); - en.add(TURNRIGHT, "right"); - en.add(TURNLEFT, "left"); - - - fr = new ArrayList(); - fr.add(STOP, "arrete:pas:voyons:merde"); - fr.add(EXPLORE, "explore"); - fr.add(FORWARD, "avance"); - fr.add(REVERSE, "recule"); - fr.add(ROTATERIGHT, "rotate&droit"); - fr.add(ROTATELEFT, "rotate&gauche"); - fr.add(TURNRIGHT, "droit"); - fr.add(TURNLEFT, "gauche"); - - - } - - public String execute(int commandInteger) { - switch (commandInteger) { - case STOP: - return stop(); - case EXPLORE: - return explore(); - case FORWARD: - return forward(); - case REVERSE: - return reverse(); - case TURNRIGHT: - return turnRight(); - case TURNLEFT: - return turnLeft(); - case ROTATERIGHT: - return rotateRight(); - case ROTATELEFT: - return rotateLeft(); - default: - return stop(); - } - } - - public String guessWhatToDo(String command) { - command = command.toLowerCase(); - int commandToExecute = STOP; - String commandForHumans = command; - - ArrayList humancommands = en; - if (language == FR) { - humancommands = fr; - } - for (int i = 0; i < humancommands.size(); i++) { - String[] andwords = humancommands.get(i).split("&"); - String[] orwords = humancommands.get(i).split(":"); - /* - * If there are AND words, then check first to see if it matches all - * words - */ - if (andwords.length > 1) { - int wordsfound = 0; - commandForHumans = andwords[0]; - for (int k = 0; k < andwords.length; k++) { - if (command.contains(andwords[k])) { - wordsfound++; - } - } - if (wordsfound >= andwords.length) { - commandToExecute = i; - return commandForHumans; - } - } - /* - * Then if a command hasn't been issued, check for the OR words. - */ - if(orwords.length > 0){ - commandForHumans = orwords[0]; - for (int k = 0; k < orwords.length; k++) { - if (command.contains(orwords[k])) { - commandToExecute = i; - return commandForHumans; - } - } - } - - - } - mCommandToExecute = commandToExecute; - return commandForHumans; - } - public String executeGuess(){ - if(mCommandToExecute >= 0){ - return execute(mCommandToExecute); - } - return ""; - } - - public RoverLexicon(int language, int timer) { - super(); - defineLanguages(); - this.language = language; - this.timer = timer; - } - public RoverLexicon(int language) { - super(); - defineLanguages(); - this.language = language; - this.timer = 5; - } - public RoverLexicon() { - super(); - defineLanguages(); - this.language = EN; - this.timer = 5; - } + private int language; + private int timer = 5; + + /* + * Languages + */ + public static final int EN = 1000; + public static final int FR = 1001; + public static final String EN_CARRIER_PHRASE = "I will tell the robot: "; + public static final String FR_CARRIER_PHRASE = "Je vais dire au robo: "; + + /* + * Commands, use the order of these constants for the precedence order of the + * commands (STOP has the highest precedence) + */ + public static final int STOP = 0; + public static final int EXPLORE = 1; + public static final int FORWARD = 2; + public static final int REVERSE = 3; + public static final int TURNRIGHT = 6; + public static final int TURNLEFT = 7; + public static final int ROTATERIGHT = 4; + public static final int ROTATELEFT = 5; + + private ArrayList en; + private ArrayList fr; + + private int mCommandToExecute; + + public String stop() { + return "S"; + } + + public String explore() { + return "E"; + } + + public String forward() { + return "1F2F"; + } + + public String reverse() { + return "1R2R"; + } + + public String turnRight() { + return "1F"; + } + + public String turnLeft() { + return "2F"; + } + + public String rotateRight() { + return "1F2R"; + } + + public String rotateLeft() { + return "1R2F"; + } + + public void defineLanguages() { + en = new ArrayList(); + en.add(STOP, "stop:wait:don't:no:damn"); + en.add(EXPLORE, "explore:try"); + en.add(FORWARD, "forward:ahead"); + en.add(REVERSE, "reverse:back"); + en.add(ROTATERIGHT, "rotate&right"); + en.add(ROTATELEFT, "rotate&left"); + en.add(TURNRIGHT, "right"); + en.add(TURNLEFT, "left"); + + fr = new ArrayList(); + fr.add(STOP, "arrête:arrêter:arrêté:arrêter:arrêtez:pas:voyons:voyant:m****:merde:zut:non:stop"); + fr.add(EXPLORE, "explore:explorer"); + fr.add(FORWARD, "avance:tu dois:tu dois:te doi:tournoi:tout droit:avanc:forward"); + fr.add(REVERSE, "recule:recul:reverse:arrière"); + fr.add(ROTATERIGHT, "pivoter vers la droite:vers la droite:pilot:pivot:rotate&droit"); + fr.add(ROTATELEFT, "pivoter vers la gauche:vers la gauche:rotate&gauche"); + fr.add(TURNRIGHT, "à la droite:droite:droit:right"); + fr.add(TURNLEFT, "à la gauche:gauche:left"); + + } + + public String execute(int commandInteger) { + switch (commandInteger) { + case STOP: + return stop(); + case EXPLORE: + return explore(); + case FORWARD: + return forward(); + case REVERSE: + return reverse(); + case TURNRIGHT: + return turnRight(); + case TURNLEFT: + return turnLeft(); + case ROTATERIGHT: + return rotateRight(); + case ROTATELEFT: + return rotateLeft(); + default: + return stop(); + } + } + + public String guessWhatToDo(String command) { + command = command.toLowerCase(); + int commandToExecute = STOP; + String commandForHumans = command; + + ArrayList humancommands = en; + if (language == FR) { + humancommands = fr; + } + for (int i = 0; i < humancommands.size(); i++) { + String[] andwords = humancommands.get(i).split("&"); + String[] orwords = humancommands.get(i).split(":"); + /* + * If there are AND words, then check first to see if it matches all words + */ + if (andwords.length > 1) { + int wordsfound = 0; + commandForHumans = andwords[0]; + for (int k = 0; k < andwords.length; k++) { + if (command.contains(andwords[k])) { + wordsfound++; + } + } + if (wordsfound >= andwords.length) { + commandToExecute = i; + return commandForHumans; + } + } + /* + * Then if a command hasn't been issued, check for the OR words. + */ + if (orwords.length > 0) { + commandForHumans = orwords[0]; + for (int k = 0; k < orwords.length; k++) { + if (command.contains(orwords[k])) { + commandToExecute = i; + return commandForHumans; + } + } + } + + } + mCommandToExecute = commandToExecute; + return commandForHumans; + } + + public String executeGuess() { + if (mCommandToExecute >= 0) { + return execute(mCommandToExecute); + } + return ""; + } + + public RoverLexicon(int language, int timer) { + super(); + defineLanguages(); + this.language = language; + this.timer = timer; + } + + public RoverLexicon(int language) { + super(); + defineLanguages(); + this.language = language; + this.timer = 5; + } + + public RoverLexicon() { + super(); + defineLanguages(); + if (Locale.getDefault().getLanguage().contains("fr")) { + this.language = FR; + } else { + this.language = EN; + } + this.timer = 5; + } } diff --git a/README.md b/README.md new file mode 100644 index 0000000..d8413ce --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +This project contains sample code for making a arduino robot recognize dots in front of it using OpenCV, send its results to MyRobots.com and also control the robot using another Android using either voice or gestures to tell the robot to turn or do other things. + +The Tutorial folder breaks it down into manageble chunks: +* [Tutorial](Tutorial) + +Issues which explain the code and let you see the diffs between each step: +* https://github.com/AndroidMontreal/UpAndRunningWithAndroid/issues?milestone=1&state=open + +The other folders are the code from our cloud robotics project (forked from google code to github since it seems to be easier for people to set up at hackathons) +* https://code.google.com/p/roogle-darwin/ + diff --git a/Tutorial/NOTICE b/Tutorial/LICENSE similarity index 100% rename from Tutorial/NOTICE rename to Tutorial/LICENSE diff --git a/Tutorial/MakeItListenAndRepeat.apk b/Tutorial/MakeItListenAndRepeat.apk index ace415b..5b08362 100644 Binary files a/Tutorial/MakeItListenAndRepeat.apk and b/Tutorial/MakeItListenAndRepeat.apk differ diff --git a/Tutorial/MakeItUnderstandVoiceAndGestures.apk b/Tutorial/MakeItUnderstandVoiceAndGestures.apk index 315ce7f..483a76e 100644 Binary files a/Tutorial/MakeItUnderstandVoiceAndGestures.apk and b/Tutorial/MakeItUnderstandVoiceAndGestures.apk differ diff --git a/Tutorial/MakeItUnderstandVoiceGesturesAndWatch.apk b/Tutorial/MakeItUnderstandVoiceGesturesAndWatch.apk new file mode 100644 index 0000000..7f63d0b Binary files /dev/null and b/Tutorial/MakeItUnderstandVoiceGesturesAndWatch.apk differ diff --git a/Tutorial/MakeItUnderstandWatch.apk b/Tutorial/MakeItUnderstandWatch.apk new file mode 100644 index 0000000..ad465e8 Binary files /dev/null and b/Tutorial/MakeItUnderstandWatch.apk differ diff --git a/Tutorial/README.md b/Tutorial/README.md new file mode 100644 index 0000000..85b3ff6 --- /dev/null +++ b/Tutorial/README.md @@ -0,0 +1,14 @@ +This is essentailly a three part tutorial to make an Android into a remote control (voice, touch or watch controlled) for a robot, garage opener, stereo etc + +The steps are in this milestone: +https://github.com/AndroidMontreal/UpAndRunningWithAndroid/milestones/Up%20and%20Running%20 + +Step one - Make it Talk + +Step two - Make it Listen + +Step three - Make it Understand + +Bonus credit: + +Step three - Control it with a watch diff --git a/Tutorial/app/build.gradle b/Tutorial/app/build.gradle new file mode 100644 index 0000000..82a2a5e --- /dev/null +++ b/Tutorial/app/build.gradle @@ -0,0 +1,26 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 18 + buildToolsVersion "23.0.0" + + defaultConfig { + applicationId "com.androidmontreal.gesturevoicecommander" + minSdkVersion 18 + compileSdkVersion 21 + targetSdkVersion 23 + } + + dependencies { + wearApp project(':wear') + compile 'com.google.android.gms:play-services:7.8.0' + compile 'watch.nudge.phonegesturelibrary:phone-gest-lib:0.8.4' + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' + } + } +} diff --git a/Tutorial/AndroidManifest.xml b/Tutorial/app/src/main/AndroidManifest.xml similarity index 56% rename from Tutorial/AndroidManifest.xml rename to Tutorial/app/src/main/AndroidManifest.xml index 30fc789..4fb7b29 100644 --- a/Tutorial/AndroidManifest.xml +++ b/Tutorial/app/src/main/AndroidManifest.xml @@ -1,4 +1,4 @@ - + - + package="com.androidmontreal.gesturevoicecommander"> + + + + - + + - + - + android:name=".GestureBuilderActivity" + android:label="@string/application_name"> - + android:label="Text To Speech"> - + android:label="Speech Recognition"> + android:label="Speech Recognition" > @@ -51,6 +50,16 @@ + + + + - \ No newline at end of file + diff --git a/Tutorial/src/com/androidmontreal/gesturevoicecommander/CreateGestureActivity.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/CreateGestureActivity.java similarity index 98% rename from Tutorial/src/com/androidmontreal/gesturevoicecommander/CreateGestureActivity.java rename to Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/CreateGestureActivity.java index 436d970..79d129a 100644 --- a/Tutorial/src/com/androidmontreal/gesturevoicecommander/CreateGestureActivity.java +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/CreateGestureActivity.java @@ -39,7 +39,7 @@ public class CreateGestureActivity extends Activity { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); - + setContentView(R.layout.create_gesture); mDoneButton = findViewById(R.id.done); @@ -51,7 +51,7 @@ protected void onCreate(Bundle savedInstanceState) { @Override protected void onSaveInstanceState(Bundle outState) { super.onSaveInstanceState(outState); - + if (mGesture != null) { outState.putParcelable("gesture", mGesture); } @@ -60,7 +60,7 @@ protected void onSaveInstanceState(Bundle outState) { @Override protected void onRestoreInstanceState(Bundle savedInstanceState) { super.onRestoreInstanceState(savedInstanceState); - + mGesture = savedInstanceState.getParcelable("gesture"); if (mGesture != null) { final GestureOverlayView overlay = @@ -99,15 +99,15 @@ public void addGesture(View v) { } finish(); - + } - + @SuppressWarnings({"UnusedDeclaration"}) public void cancelGesture(View v) { setResult(RESULT_CANCELED); finish(); } - + private class GesturesProcessor implements GestureOverlayView.OnGestureListener { public void onGestureStarted(GestureOverlayView overlay, MotionEvent event) { mDoneButton.setEnabled(false); diff --git a/Tutorial/src/com/androidmontreal/gesturevoicecommander/GestureBuilderActivity.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/GestureBuilderActivity.java similarity index 92% rename from Tutorial/src/com/androidmontreal/gesturevoicecommander/GestureBuilderActivity.java rename to Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/GestureBuilderActivity.java index 338e015..6c657db 100644 --- a/Tutorial/src/com/androidmontreal/gesturevoicecommander/GestureBuilderActivity.java +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/GestureBuilderActivity.java @@ -64,7 +64,7 @@ public class GestureBuilderActivity extends ListActivity { private static final int DIALOG_RENAME_GESTURE = 1; private static final int REQUEST_NEW_GESTURE = 1; - + // Type: long (id) private static final String GESTURES_INFO_ID = "gestures.info_id"; @@ -97,7 +97,7 @@ protected void onCreate(Bundle savedInstanceState) { if (sStore == null) { // sStore = GestureLibraries.fromFile(mStoreFile); - sStore = GestureLibraries.fromRawResource(this, R.raw.gestures); + sStore = GestureLibraries.fromRawResource(this, R.raw.gestures); } mEmpty = (TextView) findViewById(android.R.id.empty); loadGestures(); @@ -113,7 +113,7 @@ static GestureLibrary getStore() { public void reloadGestures(View v) { loadGestures(); } - + @SuppressWarnings({"UnusedDeclaration"}) public void addGesture(View v) { Intent intent = new Intent(this, CreateGestureActivity.class); @@ -123,7 +123,7 @@ public void addGesture(View v) { @Override protected void onActivityResult(int requestCode, int resultCode, Intent data) { super.onActivityResult(requestCode, resultCode, data); - + if (resultCode == RESULT_OK) { switch (requestCode) { case REQUEST_NEW_GESTURE: @@ -136,7 +136,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { private void loadGestures() { if (mTask != null && mTask.getStatus() != GesturesLoadTask.Status.FINISHED) { mTask.cancel(true); - } + } mTask = (GesturesLoadTask) new GesturesLoadTask().execute(); } @@ -174,7 +174,8 @@ protected void onRestoreInstanceState(Bundle state) { long id = state.getLong(GESTURES_INFO_ID, -1); if (id != -1) { final Set entries = sStore.getGestureEntries(); -out: for (String name : entries) { + out: + for (String name : entries) { for (Gesture gesture : sStore.getGestures(name)) { if (gesture.getID() == id) { mCurrentRenameGesture = new NamedGesture(); @@ -188,9 +189,7 @@ protected void onRestoreInstanceState(Bundle state) { } @Override - public void onCreateContextMenu(ContextMenu menu, View v, - ContextMenu.ContextMenuInfo menuInfo) { - + public void onCreateContextMenu(ContextMenu menu, View v, ContextMenu.ContextMenuInfo menuInfo) { super.onCreateContextMenu(menu, v, menuInfo); AdapterView.AdapterContextMenuInfo info = (AdapterView.AdapterContextMenuInfo) menuInfo; @@ -202,8 +201,7 @@ public void onCreateContextMenu(ContextMenu menu, View v, @Override public boolean onContextItemSelected(MenuItem item) { - final AdapterView.AdapterContextMenuInfo menuInfo = (AdapterView.AdapterContextMenuInfo) - item.getMenuInfo(); + final AdapterView.AdapterContextMenuInfo menuInfo = (AdapterView.AdapterContextMenuInfo) item.getMenuInfo(); final NamedGesture gesture = (NamedGesture) menuInfo.targetView.getTag(); switch (item.getItemId()) { @@ -254,18 +252,18 @@ public void onCancel(DialogInterface dialog) { } }); builder.setNegativeButton(getString(R.string.cancel_action), - new Dialog.OnClickListener() { - public void onClick(DialogInterface dialog, int which) { - cleanupRenameDialog(); + new Dialog.OnClickListener() { + public void onClick(DialogInterface dialog, int which) { + cleanupRenameDialog(); + } } - } ); builder.setPositiveButton(getString(R.string.rename_action), - new Dialog.OnClickListener() { - public void onClick(DialogInterface dialog, int which) { - changeGestureName(); + new Dialog.OnClickListener() { + public void onClick(DialogInterface dialog, int which) { + changeGestureName(); + } } - } ); builder.setView(layout); return builder.create(); @@ -333,8 +331,8 @@ protected void onPreExecute() { findViewById(R.id.addButton).setEnabled(false); findViewById(R.id.reloadButton).setEnabled(false); - - mAdapter.setNotifyOnChange(false); + + mAdapter.setNotifyOnChange(false); mAdapter.clear(); } @@ -352,8 +350,7 @@ protected Integer doInBackground(Void... params) { if (isCancelled()) break; for (Gesture gesture : store.getGestures(name)) { - final Bitmap bitmap = gesture.toBitmap(mThumbnailSize, mThumbnailSize, - mThumbnailInset, mPathColor); + final Bitmap bitmap = gesture.toBitmap(mThumbnailSize, mThumbnailSize, mThumbnailInset, mPathColor); final NamedGesture namedGesture = new NamedGesture(); namedGesture.gesture = gesture; namedGesture.name = name; @@ -391,8 +388,7 @@ protected void onPostExecute(Integer result) { if (result == STATUS_NO_STORAGE) { getListView().setVisibility(View.GONE); mEmpty.setVisibility(View.VISIBLE); - mEmpty.setText(getString(R.string.gestures_error_loading, - mStoreFile.getAbsolutePath())); + mEmpty.setText(getString(R.string.gestures_error_loading, mStoreFile.getAbsolutePath())); } else { findViewById(R.id.addButton).setEnabled(true); findViewById(R.id.reloadButton).setEnabled(true); @@ -408,8 +404,7 @@ static class NamedGesture { private class GesturesAdapter extends ArrayAdapter { private final LayoutInflater mInflater; - private final Map mThumbnails = Collections.synchronizedMap( - new HashMap()); + private final Map mThumbnails = Collections.synchronizedMap(new HashMap()); public GesturesAdapter(Context context) { super(context, 0); @@ -431,8 +426,7 @@ public View getView(int position, View convertView, ViewGroup parent) { label.setTag(gesture); label.setText(gesture.name); - label.setCompoundDrawablesWithIntrinsicBounds(mThumbnails.get(gesture.gesture.getID()), - null, null, null); + label.setCompoundDrawablesWithIntrinsicBounds(mThumbnails.get(gesture.gesture.getID()), null, null, null); return convertView; } diff --git a/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItListenAndRepeat.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItListenAndRepeat.java new file mode 100644 index 0000000..73920cb --- /dev/null +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItListenAndRepeat.java @@ -0,0 +1,157 @@ +package com.androidmontreal.gesturevoicecommander.practice; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import android.app.Activity; +import android.content.Context; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.content.pm.ResolveInfo; +import android.os.Bundle; +import android.speech.RecognizerIntent; +import android.speech.tts.TextToSpeech; +import android.speech.tts.TextToSpeech.OnInitListener; +import android.util.Log; +import android.widget.Toast; + +import com.androidmontreal.gesturevoicecommander.R; + +/** + * Building on what we saw in MakeItTalk, now lets make it Listen. Here is some + * super simple code that uses the VoiceRecognition Intent to recognize what the + * user says, and then uses Text To Speech to tell the user what it might have + * heard. + * + * @author cesine + */ +public class MakeItListenAndRepeat extends Activity implements OnInitListener { + private static final String TAG = "MakeItListen"; + private static final int RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE = 341; + /** + * Talk to the user + */ + private TextToSpeech mTts; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + mTts = new TextToSpeech(this, this); + } + + protected void promptTheUserToTalk() { + if (isIntentAvailable(RecognizerIntent.ACTION_RECOGNIZE_SPEECH)) { + this.speak(getString(R.string.im_listening)); + } else { + this.speak(getString(R.string.i_cant_listen)); + } + } + + /** + * Fire an intent to start the voice recognition activity. + */ + private void startVoiceRecognitionActivity() { + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); + intent.putExtra(RecognizerIntent.EXTRA_PROMPT, getString(R.string.im_listening)); + if (isIntentAvailable(intent)) { + startActivityForResult(intent, RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE); + } else { + Log.w(TAG, "This device doesn't have speech recognition, maybe its an emulator or a phone from china without google products?"); + } + } + + /** + * Handle the results from the voice recognition activity. + */ + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + if (requestCode == RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK) { + /* + * Populate the wordsList with the String values the recognition engine + * thought it heard, and then Toast them to the user and say them out + * loud. + */ + ArrayList matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS); + for (int iMightHaveHeardThis = 0; iMightHaveHeardThis < matches.size(); iMightHaveHeardThis++) { + + /* Build a carrierPhrase if you want it to make some sense */ + String carrierPhrase = getString(R.string.i_might_have_heard); + if (iMightHaveHeardThis > 0) { + carrierPhrase = getString(R.string.or_maybe); + } + carrierPhrase += " " + matches.get(iMightHaveHeardThis) + "."; + + Toast.makeText(this, carrierPhrase, Toast.LENGTH_LONG).show(); + this.speak(carrierPhrase); + + /* + * Don't go on forever, it there are too many potential matches don't + * say them all + */ + if (iMightHaveHeardThis == 2 && matches.size() > 2) { + this.speak(getString(R.string.there_were_others)); + break; + } + } + } + super.onActivityResult(requestCode, resultCode, data); + } + + @Override + protected void onDestroy() { + if (mTts != null) { + mTts.stop(); + mTts.shutdown(); + } + super.onDestroy(); + } + + @Override + public void onInit(int status) { + if (status == TextToSpeech.SUCCESS) { + int result = mTts.setLanguage(Locale.getDefault()); + if (result == TextToSpeech.LANG_MISSING_DATA || result == TextToSpeech.LANG_NOT_SUPPORTED) { + Log.e(TAG, "Language is not available."); + Toast.makeText(this, + "The " + Locale.getDefault().getDisplayLanguage() + + " TextToSpeech isn't installed, you can go into the " + + "\nAndroid's settings in the " + + "\nVoice Input and Output menu to turn it on. ", + Toast.LENGTH_LONG).show(); + } else { + // everything is working. + promptTheUserToTalk(); + startVoiceRecognitionActivity(); + } + } else { + Toast.makeText(this, + "Sorry, I can't talk to you because " + + "I could not initialize TextToSpeech.", Toast.LENGTH_LONG) + .show(); + } + } + + public boolean speak(String message) { + if (mTts != null) { + mTts.speak(message, TextToSpeech.QUEUE_ADD, null); + } else { + Toast.makeText(this, "Sorry, I can't speak to you: " + message, Toast.LENGTH_LONG).show(); + } + return true; + } + + public boolean isIntentAvailable(String action) { + final Intent intent = new Intent(action); + return isIntentAvailable(intent); + } + + public boolean isIntentAvailable(final Intent intent) { + final PackageManager packageManager = this.getPackageManager(); + List list = packageManager.queryIntentActivities(intent, PackageManager.MATCH_DEFAULT_ONLY); + return list.size() > 0; + } + +} diff --git a/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItTalk.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItTalk.java new file mode 100644 index 0000000..fbb96c4 --- /dev/null +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItTalk.java @@ -0,0 +1,64 @@ +package com.androidmontreal.gesturevoicecommander.practice; + +import java.util.Locale; + +import android.app.Activity; +import android.os.Bundle; +import android.speech.tts.TextToSpeech; +import android.speech.tts.TextToSpeech.OnInitListener; +import android.util.Log; +import android.widget.Toast; + +import com.androidmontreal.gesturevoicecommander.R; + +public class MakeItTalk extends Activity implements OnInitListener { + private static final String TAG = "MakeItTalk"; + /** + * Talk to the user + */ + private TextToSpeech mTts; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + mTts = new TextToSpeech(this, this); + } + + protected void sayFirstWords() { + mTts.speak(getString(R.string.my_first_words), TextToSpeech.QUEUE_ADD, null); + } + + @Override + protected void onDestroy() { + if (mTts != null) { + mTts.stop(); + mTts.shutdown(); + } + super.onDestroy(); + } + + @Override + public void onInit(int status) { + if (status == TextToSpeech.SUCCESS) { + int result = mTts.setLanguage(Locale.getDefault()); + if (result == TextToSpeech.LANG_MISSING_DATA || result == TextToSpeech.LANG_NOT_SUPPORTED) { + Log.e(TAG, "Language is not available."); + Toast.makeText(this, + "The " + Locale.getDefault().getDisplayLanguage() + + " TextToSpeech isn't installed, you can go into the " + + "\nAndroid's settings in the " + + "\nVoice Input and Output menu to turn it on. ", + Toast.LENGTH_LONG).show(); + } else { + // everything is working. + sayFirstWords(); + } + } else { + Toast.makeText(this, + "Sorry, I can't talk to you because " + + "I could not initialize TextToSpeech.", Toast.LENGTH_LONG) + .show(); + } + } +} diff --git a/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItUnderstandGestures.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItUnderstandGestures.java new file mode 100644 index 0000000..0ef4526 --- /dev/null +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/practice/MakeItUnderstandGestures.java @@ -0,0 +1,240 @@ +package com.androidmontreal.gesturevoicecommander.practice; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import android.app.Activity; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.content.pm.ResolveInfo; +import android.gesture.Gesture; +import android.gesture.GestureLibraries; +import android.gesture.GestureLibrary; +import android.gesture.GestureOverlayView; +import android.gesture.Prediction; +import android.gesture.GestureOverlayView.OnGesturePerformedListener; +import android.os.Bundle; +import android.speech.RecognizerIntent; +import android.speech.tts.TextToSpeech; +import android.speech.tts.TextToSpeech.OnInitListener; +import android.util.Log; +import android.view.View; +import android.widget.Toast; + +import com.androidmontreal.gesturevoicecommander.GestureBuilderActivity; +import com.androidmontreal.gesturevoicecommander.R; +import com.androidmontreal.gesturevoicecommander.robots.Lexicon; + +import watch.nudge.phonegesturelibrary.AbstractPhoneGestureActivity; + +/** + * Building on what we saw in MakeItListenAndRepeat, now lets make it understand + * gestures, or speech (sometimes its too noisy or too public to speak to your + * Android). Here is some super simple code that builds on the GestureBuilder + * sample code to recognize what the user wants the Android to do, and then use + * Text To Speech to tell the user what it might have understood. + * + * @author cesine + */ +public class MakeItUnderstandGestures extends AbstractPhoneGestureActivity implements OnInitListener, OnGesturePerformedListener { + private static final String TAG = "MakeItUnderstandGesture"; + private static final int RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE = 341; + private static final boolean D = true; + + /** + * Talk to the user + */ + private TextToSpeech mTts; + + /* + * A gesture library we created with the GestureBuilder, saved on the SDCard + * and then imported into the res/raw folder of this project + */ + private GestureLibrary gestureLib; + + /* A little lexicon we made for the DFR Rover at Cloud Robotics Hackathon */ + private Lexicon lexicon; + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + mTts = new TextToSpeech(this, this); + + GestureOverlayView gestureOverlayView = new GestureOverlayView(this); + View inflate = getLayoutInflater().inflate(R.layout.commander, null); + gestureOverlayView.addView(inflate); + gestureOverlayView.addOnGesturePerformedListener(this); + // gestureLib = GestureLibraries.fromFile(fileOnYourSDCard); + gestureLib = GestureLibraries.fromRawResource(this, R.raw.gestures); + if (!gestureLib.load()) { + finish(); + } + setContentView(gestureOverlayView); + + lexicon = new Lexicon(); + } + + protected void promptTheUserToTalk() { + if (isIntentAvailable(RecognizerIntent.ACTION_RECOGNIZE_SPEECH)) { + this.speak(getString(R.string.im_listening)); + } else { + this.speak(getString(R.string.i_cant_listen)); + } + } + + /** + * Fire an intent to start the voice recognition activity. + */ + private void startVoiceRecognitionActivity() { + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); + intent.putExtra(RecognizerIntent.EXTRA_PROMPT, getString(R.string.im_listening)); + if (isIntentAvailable(intent)) { + startActivityForResult(intent, RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE); + } else { + Log.w(TAG, "This device doesn't have speech recognition, maybe its an emulator or a phone from china without google products?"); + } + } + + /** + * Handle the results from the voice recognition activity. + */ + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + if (requestCode == RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE && resultCode == RESULT_OK) { + ArrayList matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS); + /* try to find a robot command in the first match */ + if (matches.size() > 0) { + sendRobotThisCommand(matches.get(0)); + } + } + super.onActivityResult(requestCode, resultCode, data); + } + + @Override + public void onDestroy() { + if (mTts != null) { + mTts.stop(); + mTts.shutdown(); + } + super.onDestroy(); + } + + @Override + public void onInit(int status) { + if (status == TextToSpeech.SUCCESS) { + int result = mTts.setLanguage(Locale.getDefault()); + if (result == TextToSpeech.LANG_MISSING_DATA || result == TextToSpeech.LANG_NOT_SUPPORTED) { + Log.e(TAG, "Language is not available."); + Toast.makeText(this, + "The " + Locale.getDefault().getDisplayLanguage() + + " TextToSpeech isn't installed, you can go into the " + + "\nAndroid's settings in the " + + "\nVoice Input and Output menu to turn it on. ", + Toast.LENGTH_LONG).show(); + } else { + // everything is working. + this.speak(getString(R.string.instructions_to_look_at_menu)); + } + } else { + Toast.makeText(this, "Sorry, I can't talk to you because " + + "I could not initialize TextToSpeech.", Toast.LENGTH_LONG).show(); + } + } + + public boolean speak(String message) { + if (mTts != null) { + mTts.speak(message, TextToSpeech.QUEUE_ADD, null); + } else { + Toast.makeText(this, "Sorry, I can't speak to you: " + message, Toast.LENGTH_LONG).show(); + } + return true; + } + + @Override + public void onGesturePerformed(GestureOverlayView overlay, Gesture gesture) { + ArrayList predictions = gestureLib.recognize(gesture); + for (Prediction prediction : predictions) { + if (prediction.score > 3.0) { + Log.d(TAG, "Detected this gesture " + prediction.name + " with a score of " + prediction.score); + } + } + if (predictions.size() > 0) { + sendRobotThisCommand(predictions.get(0).name); + } + } + @Override + public void onSnap() { + sendRobotThisCommand(lexicon.stop()); + } + + @Override + public void onFlick() { + sendRobotThisCommand(lexicon.explore()); + } + + @Override + public void onTwist() { + sendRobotThisCommand(lexicon.rotateRight()); + } + +//These functions won't be called until you subscribe to the appropriate gestures +//in a class that extends AbstractGestureClientActivity in a wear app. + + @Override + public void onTiltX(float x) { + Log.e(TAG, "This function should not be called unless subscribed to TILT_X " + x); + if (x < 0){ + sendRobotThisCommand(lexicon.turnLeft()); + } else { + sendRobotThisCommand(lexicon.turnRight()); + } +// throw new IllegalStateException("This function should not be called unless subscribed to TILT_X."); + } + + @Override + public void onTilt(float x, float y, float z) { + Log.e(TAG, "This function should not be called unless subscribed to onTilt." + x + " " + y + " " + z); + } + + @Override + public void onWindowClosed() { + Log.e("MainWatchActivity","This function should not be called unless windowed gesture detection is enabled."); + } + + + public String sendRobotThisCommand(String command) { + String guessedCommand = lexicon.guessWhatToDo(command); + Toast.makeText(this, guessedCommand, Toast.LENGTH_SHORT).show(); + + if (Locale.getDefault().getLanguage().contains("fr")) { + mTts.speak(lexicon.FR_CARRIER_PHRASE + guessedCommand, TextToSpeech.QUEUE_ADD, null); + } else { + mTts.speak(lexicon.EN_CARRIER_PHRASE + guessedCommand, TextToSpeech.QUEUE_ADD, null); + } + return lexicon.executeGuess(); + } + + public void onCommandByVoiceClick(View v) { + promptTheUserToTalk(); + startVoiceRecognitionActivity(); + } + + public void onViewGesturesClick(View v) { + Intent i = new Intent(this, GestureBuilderActivity.class); + startActivity(i); + } + + public boolean isIntentAvailable(String action) { + final Intent intent = new Intent(action); + return isIntentAvailable(intent); + } + + public boolean isIntentAvailable(final Intent intent) { + final PackageManager packageManager = this.getPackageManager(); + List list = packageManager.queryIntentActivities(intent, PackageManager.MATCH_DEFAULT_ONLY); + return list.size() > 0; + } +} diff --git a/AndroidArduinoOpenCV/src/com/androidmontreal/arduino/bluetooth/BluetoothChatService.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/BluetoothChatService.java similarity index 56% rename from AndroidArduinoOpenCV/src/com/androidmontreal/arduino/bluetooth/BluetoothChatService.java rename to Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/BluetoothChatService.java index ba7a8f7..315875f 100644 --- a/AndroidArduinoOpenCV/src/com/androidmontreal/arduino/bluetooth/BluetoothChatService.java +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/BluetoothChatService.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009 The Android Open Source Project + * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,7 @@ * limitations under the License. */ -package com.androidmontreal.arduino.bluetooth; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.UUID; +package com.androidmontreal.gesturevoicecommander.robots; import android.bluetooth.BluetoothAdapter; import android.bluetooth.BluetoothDevice; @@ -31,51 +26,62 @@ import android.os.Message; import android.util.Log; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.UUID; + /** * This class does all the work for setting up and managing Bluetooth - * connections with other devices. It has a thread that listens for incoming - * connections, a thread for connecting with a device, and a thread for - * performing data transmissions when connected. + * connections with other devices. It has a thread that listens for + * incoming connections, a thread for connecting with a device, and a + * thread for performing data transmissions when connected. */ public class BluetoothChatService { // Debugging private static final String TAG = "BluetoothChatService"; - private static final boolean D = true; // Name for the SDP record when creating server socket - private static final String NAME = "BluetoothChat"; + private static final String NAME_SECURE = "BluetoothChatSecure"; + private static final String NAME_INSECURE = "BluetoothChatInsecure"; // Unique UUID for this application - private static final UUID MY_UUID = UUID - .fromString("fa87c0d0-afac-11de-8a39-0800200c9a66"); - - private static final UUID SERIAL_PORT_PROFILE_DEFAULT_UUID = UUID - .fromString("00001101-0000-1000-8000-0005F9B34FB"); + private static final UUID MY_UUID_SECURE = + UUID.fromString("fa87c0d0-afac-11de-8a39-0800200c9a66"); + private static final UUID MY_UUID_INSECURE = + UUID.fromString("8ce255c0-200a-11e0-ac64-0800200c9a66"); // Member fields private final BluetoothAdapter mAdapter; private final Handler mHandler; - private AcceptThread mAcceptThread; + private AcceptThread mSecureAcceptThread; + private AcceptThread mInsecureAcceptThread; private ConnectThread mConnectThread; private ConnectedThread mConnectedThread; private int mState; // Constants that indicate the current connection state - public static final int STATE_NONE = 0; // we're doing nothing - public static final int STATE_LISTEN = 1; // now listening for incoming - // connections - public static final int STATE_CONNECTING = 2; // now initiating an outgoing - // connection - public static final int STATE_CONNECTED = 3; // now connected to a remote - // device + public static final int STATE_NONE = 0; // we're doing nothing + public static final int STATE_LISTEN = 1; // now listening for incoming connections + public static final int STATE_CONNECTING = 2; // now initiating an outgoing connection + public static final int STATE_CONNECTED = 3; // now connected to a remote device + + // Message types sent from the BluetoothChatService Handler + public static final int MESSAGE_STATE_CHANGE = 1; + public static final int MESSAGE_READ = 2; + public static final int MESSAGE_WRITE = 3; + public static final int MESSAGE_DEVICE_NAME = 4; + public static final int MESSAGE_TOAST = 5; + + // Key names received from the BluetoothChatService Handler + public static final String DEVICE_NAME = "device_name"; + public static final String TOAST = "toast"; /** * Constructor. Prepares a new BluetoothChat session. - * - * @param context - * The UI Activity Context - * @param handler - * A Handler to send messages back to the UI Activity + * + * @param context The UI Activity Context + * @param handler A Handler to send messages back to the UI Activity */ public BluetoothChatService(Context context, Handler handler) { mAdapter = BluetoothAdapter.getDefaultAdapter(); @@ -85,18 +91,15 @@ public BluetoothChatService(Context context, Handler handler) { /** * Set the current state of the chat connection - * - * @param state - * An integer defining the current connection state + * + * @param state An integer defining the current connection state */ private synchronized void setState(int state) { - if (D) - Log.d(TAG, "setState() " + mState + " -> " + state); + Log.d(TAG, "setState() " + mState + " -> " + state); mState = state; // Give the new state to the Handler so the UI Activity can update - mHandler.obtainMessage(RoogleTank.MESSAGE_STATE_CHANGE, state, -1) - .sendToTarget(); + mHandler.obtainMessage(MESSAGE_STATE_CHANGE, state, -1).sendToTarget(); } /** @@ -111,8 +114,7 @@ public synchronized int getState() { * session in listening (server) mode. Called by the Activity onResume() */ public synchronized void start() { - if (D) - Log.d(TAG, "start"); + Log.d(TAG, "start"); // Cancel any thread attempting to make a connection if (mConnectThread != null) { @@ -126,23 +128,27 @@ public synchronized void start() { mConnectedThread = null; } + setState(STATE_LISTEN); + // Start the thread to listen on a BluetoothServerSocket - if (mAcceptThread == null) { - mAcceptThread = new AcceptThread(); - mAcceptThread.start(); + if (mSecureAcceptThread == null) { + mSecureAcceptThread = new AcceptThread(true); + mSecureAcceptThread.start(); + } + if (mInsecureAcceptThread == null) { + mInsecureAcceptThread = new AcceptThread(false); + mInsecureAcceptThread.start(); } - setState(STATE_LISTEN); } /** * Start the ConnectThread to initiate a connection to a remote device. - * - * @param device - * The BluetoothDevice to connect + * + * @param device The BluetoothDevice to connect + * @param secure Socket Security type - Secure (true) , Insecure (false) */ - public synchronized void connect(BluetoothDevice device) { - if (D) - Log.d(TAG, "connect to: " + device); + public synchronized void connect(BluetoothDevice device, boolean secure) { + Log.d(TAG, "connect to: " + device); // Cancel any thread attempting to make a connection if (mState == STATE_CONNECTING) { @@ -159,23 +165,20 @@ public synchronized void connect(BluetoothDevice device) { } // Start the thread to connect with the given device - mConnectThread = new ConnectThread(device); + mConnectThread = new ConnectThread(device, secure); mConnectThread.start(); setState(STATE_CONNECTING); } /** * Start the ConnectedThread to begin managing a Bluetooth connection - * - * @param socket - * The BluetoothSocket on which the connection was made - * @param device - * The BluetoothDevice that has been connected + * + * @param socket The BluetoothSocket on which the connection was made + * @param device The BluetoothDevice that has been connected */ - public synchronized void connected(BluetoothSocket socket, - BluetoothDevice device) { - if (D) - Log.d(TAG, "connected"); + public synchronized void connected(BluetoothSocket socket, BluetoothDevice + device, final String socketType) { + Log.d(TAG, "connected, Socket Type:" + socketType); // Cancel the thread that completed the connection if (mConnectThread != null) { @@ -189,21 +192,24 @@ public synchronized void connected(BluetoothSocket socket, mConnectedThread = null; } - // Cancel the accept thread because we only want to connect to one - // device - if (mAcceptThread != null) { - mAcceptThread.cancel(); - mAcceptThread = null; + // Cancel the accept thread because we only want to connect to one device + if (mSecureAcceptThread != null) { + mSecureAcceptThread.cancel(); + mSecureAcceptThread = null; + } + if (mInsecureAcceptThread != null) { + mInsecureAcceptThread.cancel(); + mInsecureAcceptThread = null; } // Start the thread to manage the connection and perform transmissions - mConnectedThread = new ConnectedThread(socket); + mConnectedThread = new ConnectedThread(socket, socketType); mConnectedThread.start(); // Send the name of the connected device back to the UI Activity - Message msg = mHandler.obtainMessage(RoogleTank.MESSAGE_DEVICE_NAME); + Message msg = mHandler.obtainMessage(MESSAGE_DEVICE_NAME); Bundle bundle = new Bundle(); - bundle.putString(RoogleTank.DEVICE_NAME, device.getName()); + bundle.putString(DEVICE_NAME, device.getName()); msg.setData(bundle); mHandler.sendMessage(msg); @@ -214,28 +220,34 @@ public synchronized void connected(BluetoothSocket socket, * Stop all threads */ public synchronized void stop() { - if (D) - Log.d(TAG, "stop"); + Log.d(TAG, "stop"); + if (mConnectThread != null) { mConnectThread.cancel(); mConnectThread = null; } + if (mConnectedThread != null) { mConnectedThread.cancel(); mConnectedThread = null; } - if (mAcceptThread != null) { - mAcceptThread.cancel(); - mAcceptThread = null; + + if (mSecureAcceptThread != null) { + mSecureAcceptThread.cancel(); + mSecureAcceptThread = null; + } + + if (mInsecureAcceptThread != null) { + mInsecureAcceptThread.cancel(); + mInsecureAcceptThread = null; } setState(STATE_NONE); } /** * Write to the ConnectedThread in an unsynchronized manner - * - * @param out - * The bytes to write + * + * @param out The bytes to write * @see ConnectedThread#write(byte[]) */ public void write(byte[] out) { @@ -243,8 +255,7 @@ public void write(byte[] out) { ConnectedThread r; // Synchronize a copy of the ConnectedThread synchronized (this) { - if (mState != STATE_CONNECTED) - return; + if (mState != STATE_CONNECTED) return; r = mConnectedThread; } // Perform the write unsynchronized @@ -255,56 +266,66 @@ public void write(byte[] out) { * Indicate that the connection attempt failed and notify the UI Activity. */ private void connectionFailed() { - setState(STATE_LISTEN); - // Send a failure message back to the Activity - Message msg = mHandler.obtainMessage(RoogleTank.MESSAGE_TOAST); + Message msg = mHandler.obtainMessage(MESSAGE_TOAST); Bundle bundle = new Bundle(); - bundle.putString(RoogleTank.TOAST, "Unable to connect device"); + bundle.putString(TOAST, "Unable to connect device"); msg.setData(bundle); mHandler.sendMessage(msg); + + // Start the service over to restart listening mode + BluetoothChatService.this.start(); } /** * Indicate that the connection was lost and notify the UI Activity. */ private void connectionLost() { - setState(STATE_LISTEN); - // Send a failure message back to the Activity - Message msg = mHandler.obtainMessage(RoogleTank.MESSAGE_TOAST); + Message msg = mHandler.obtainMessage(MESSAGE_TOAST); Bundle bundle = new Bundle(); - bundle.putString(RoogleTank.TOAST, "Device connection was lost"); + bundle.putString(TOAST, "Device connection was lost"); msg.setData(bundle); mHandler.sendMessage(msg); + + // Start the service over to restart listening mode + BluetoothChatService.this.start(); } /** * This thread runs while listening for incoming connections. It behaves - * like a server-side client. It runs until a connection is accepted (or - * until cancelled). + * like a server-side client. It runs until a connection is accepted + * (or until cancelled). */ private class AcceptThread extends Thread { // The local server socket private final BluetoothServerSocket mmServerSocket; + private String mSocketType; - public AcceptThread() { + public AcceptThread(boolean secure) { BluetoothServerSocket tmp = null; + mSocketType = secure ? "Secure" : "Insecure"; // Create a new listening server socket try { - tmp = mAdapter - .listenUsingRfcommWithServiceRecord(NAME, SERIAL_PORT_PROFILE_DEFAULT_UUID); + if (secure) { + tmp = mAdapter.listenUsingRfcommWithServiceRecord(NAME_SECURE, + MY_UUID_SECURE); + } else { + tmp = mAdapter.listenUsingInsecureRfcommWithServiceRecord( + NAME_INSECURE, MY_UUID_INSECURE); + } } catch (IOException e) { - Log.e(TAG, "listen() failed", e); + Log.e(TAG, "Socket Type: " + mSocketType + "listen() failed", e); } mmServerSocket = tmp; } public void run() { - if (D) - Log.d(TAG, "BEGIN mAcceptThread" + this); - setName("AcceptThread"); + Log.d(TAG, "Socket Type: " + mSocketType + + "BEGIN mAcceptThread" + this); + setName("AcceptThread" + mSocketType); + BluetoothSocket socket = null; // Listen to the server socket if we're not connected @@ -314,7 +335,8 @@ public void run() { // successful connection or an exception socket = mmServerSocket.accept(); } catch (IOException e) { - Log.e(TAG, "accept() failed", e); + Log.e(TAG, "Socket Type: " + mSocketType + "accept() failed", e); + e.printStackTrace(); break; } @@ -322,66 +344,75 @@ public void run() { if (socket != null) { synchronized (BluetoothChatService.this) { switch (mState) { - case STATE_LISTEN: - case STATE_CONNECTING: - // Situation normal. Start the connected thread. - connected(socket, socket.getRemoteDevice()); - break; - case STATE_NONE: - case STATE_CONNECTED: - // Either not ready or already connected. Terminate - // new socket. - try { - socket.close(); - } catch (IOException e) { - Log.e(TAG, "Could not close unwanted socket", e); - } - break; + case STATE_LISTEN: + case STATE_CONNECTING: + // Situation normal. Start the connected thread. + connected(socket, socket.getRemoteDevice(), + mSocketType); + break; + case STATE_NONE: + case STATE_CONNECTED: + // Either not ready or already connected. Terminate new socket. + try { + socket.close(); + } catch (IOException e) { + Log.e(TAG, "Could not close unwanted socket", e); + } + break; } } } } - if (D) - Log.i(TAG, "END mAcceptThread"); + Log.i(TAG, "END mAcceptThread, socket Type: " + mSocketType); + } public void cancel() { - if (D) - Log.d(TAG, "cancel " + this); + Log.d(TAG, "Socket Type" + mSocketType + "cancel " + this); try { mmServerSocket.close(); } catch (IOException e) { - Log.e(TAG, "close() of server failed", e); + Log.e(TAG, "Socket Type" + mSocketType + "close() of server failed", e); } } } + /** - * This thread runs while attempting to make an outgoing connection with a - * device. It runs straight through; the connection either succeeds or - * fails. + * This thread runs while attempting to make an outgoing connection + * with a device. It runs straight through; the connection either + * succeeds or fails. */ private class ConnectThread extends Thread { private final BluetoothSocket mmSocket; private final BluetoothDevice mmDevice; + private String mSocketType; - public ConnectThread(BluetoothDevice device) { + public ConnectThread(BluetoothDevice device, boolean secure) { mmDevice = device; BluetoothSocket tmp = null; + mSocketType = secure ? "Secure" : "Insecure"; // Get a BluetoothSocket for a connection with the // given BluetoothDevice try { - tmp = device.createRfcommSocketToServiceRecord(SERIAL_PORT_PROFILE_DEFAULT_UUID); + if (secure) { + tmp = device.createRfcommSocketToServiceRecord( + MY_UUID_SECURE); + } else { + tmp = device.createInsecureRfcommSocketToServiceRecord( + MY_UUID_INSECURE); + } } catch (IOException e) { - Log.e(TAG, "create() failed", e); + Log.e(TAG, "Socket Type: " + mSocketType + "create() failed", e); + e.printStackTrace(); } mmSocket = tmp; } public void run() { - Log.i(TAG, "BEGIN mConnectThread"); - setName("ConnectThread"); + Log.i(TAG, "BEGIN mConnectThread SocketType:" + mSocketType); + setName("ConnectThread" + mSocketType); // Always cancel discovery because it will slow down a connection mAdapter.cancelDiscovery(); @@ -392,17 +423,17 @@ public void run() { // successful connection or an exception mmSocket.connect(); } catch (IOException e) { - connectionFailed(); + Log.d(TAG, "exception when trying to connect"); + e.printStackTrace(); // Close the socket try { mmSocket.close(); } catch (IOException e2) { - Log.e(TAG, - "unable to close() socket during connection failure", - e2); + Log.e(TAG, "unable to close() " + mSocketType + + " socket during connection failure", e2); + e2.printStackTrace(); } - // Start the service over to restart listening mode - BluetoothChatService.this.start(); + connectionFailed(); return; } @@ -412,29 +443,29 @@ public void run() { } // Start the connected thread - connected(mmSocket, mmDevice); + connected(mmSocket, mmDevice, mSocketType); } public void cancel() { try { mmSocket.close(); } catch (IOException e) { - Log.e(TAG, "close() of connect socket failed", e); + Log.e(TAG, "close() of connect " + mSocketType + " socket failed", e); } } } /** - * This thread runs during a connection with a remote device. It handles all - * incoming and outgoing transmissions. + * This thread runs during a connection with a remote device. + * It handles all incoming and outgoing transmissions. */ private class ConnectedThread extends Thread { private final BluetoothSocket mmSocket; private final InputStream mmInStream; private final OutputStream mmOutStream; - public ConnectedThread(BluetoothSocket socket) { - Log.d(TAG, "create ConnectedThread"); + public ConnectedThread(BluetoothSocket socket, String socketType) { + Log.d(TAG, "create ConnectedThread: " + socketType); mmSocket = socket; InputStream tmpIn = null; OutputStream tmpOut = null; @@ -463,11 +494,13 @@ public void run() { bytes = mmInStream.read(buffer); // Send the obtained bytes to the UI Activity - mHandler.obtainMessage(RoogleTank.MESSAGE_READ, bytes, - -1, buffer).sendToTarget(); + mHandler.obtainMessage(MESSAGE_READ, bytes, -1, buffer) + .sendToTarget(); } catch (IOException e) { Log.e(TAG, "disconnected", e); connectionLost(); + // Start the service over to restart listening mode + BluetoothChatService.this.start(); break; } } @@ -475,17 +508,16 @@ public void run() { /** * Write to the connected OutStream. - * - * @param buffer - * The bytes to write + * + * @param buffer The bytes to write */ public void write(byte[] buffer) { try { mmOutStream.write(buffer); // Share the sent message back to the UI Activity - mHandler.obtainMessage(RoogleTank.MESSAGE_WRITE, -1, -1, - buffer).sendToTarget(); + mHandler.obtainMessage(MESSAGE_WRITE, -1, -1, buffer) + .sendToTarget(); } catch (IOException e) { Log.e(TAG, "Exception during write", e); } diff --git a/AndroidArduinoOpenCV/src/com/androidmontreal/arduino/bluetooth/DeviceListActivity.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/DeviceListActivity.java similarity index 69% rename from AndroidArduinoOpenCV/src/com/androidmontreal/arduino/bluetooth/DeviceListActivity.java rename to Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/DeviceListActivity.java index 31cec9b..bd74255 100644 --- a/AndroidArduinoOpenCV/src/com/androidmontreal/arduino/bluetooth/DeviceListActivity.java +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/DeviceListActivity.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009 The Android Open Source Project + * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,9 +14,7 @@ * limitations under the License. */ -package com.androidmontreal.arduino.bluetooth; - -import java.util.Set; +package com.androidmontreal.gesturevoicecommander.robots; import android.app.Activity; import android.bluetooth.BluetoothAdapter; @@ -29,15 +27,16 @@ import android.util.Log; import android.view.View; import android.view.Window; -import android.view.View.OnClickListener; import android.widget.AdapterView; import android.widget.ArrayAdapter; import android.widget.Button; import android.widget.ListView; import android.widget.TextView; -import android.widget.AdapterView.OnItemClickListener; +import android.widget.Toast; + +import com.androidmontreal.gesturevoicecommander.R; -import com.androidmontreal.opencv.R; +import java.util.Set; /** * This Activity appears as a dialog. It lists any paired devices and @@ -46,16 +45,30 @@ * Activity in the result Intent. */ public class DeviceListActivity extends Activity { - // Debugging + + /** + * Tag for Log + */ private static final String TAG = "DeviceListActivity"; - private static final boolean D = true; - // Return Intent extra + /** + * Request codes + */ + private static final int REQUEST_ENABLE_BT = 9234; + + /** + * Return Intent extra + */ public static String EXTRA_DEVICE_ADDRESS = "device_address"; - // Member fields + /** + * Member fields + */ private BluetoothAdapter mBtAdapter; - private ArrayAdapter mPairedDevicesArrayAdapter; + + /** + * Newly discovered devices + */ private ArrayAdapter mNewDevicesArrayAdapter; @Override @@ -64,14 +77,27 @@ protected void onCreate(Bundle savedInstanceState) { // Setup the window requestWindowFeature(Window.FEATURE_INDETERMINATE_PROGRESS); - setContentView(R.layout.device_list); + setContentView(R.layout.activity_device_list); + } - // Set result CANCELED incase the user backs out + @Override + protected void onResume() { + super.onResume(); + + // Set result CANCELED in case the user backs out setResult(Activity.RESULT_CANCELED); + // Get the local Bluetooth adapter + mBtAdapter = BluetoothAdapter.getDefaultAdapter(); + if (mBtAdapter == null){ + blueToothIsNotSupported(); + return; + } + requestBluetooth(); + // Initialize the button to perform device discovery Button scanButton = (Button) findViewById(R.id.button_scan); - scanButton.setOnClickListener(new OnClickListener() { + scanButton.setOnClickListener(new View.OnClickListener() { public void onClick(View v) { doDiscovery(); v.setVisibility(View.GONE); @@ -80,12 +106,13 @@ public void onClick(View v) { // Initialize array adapters. One for already paired devices and // one for newly discovered devices - mPairedDevicesArrayAdapter = new ArrayAdapter(this, R.layout.device_name); + ArrayAdapter pairedDevicesArrayAdapter = + new ArrayAdapter(this, R.layout.device_name); mNewDevicesArrayAdapter = new ArrayAdapter(this, R.layout.device_name); // Find and set up the ListView for paired devices ListView pairedListView = (ListView) findViewById(R.id.paired_devices); - pairedListView.setAdapter(mPairedDevicesArrayAdapter); + pairedListView.setAdapter(pairedDevicesArrayAdapter); pairedListView.setOnItemClickListener(mDeviceClickListener); // Find and set up the ListView for newly discovered devices @@ -101,9 +128,6 @@ public void onClick(View v) { filter = new IntentFilter(BluetoothAdapter.ACTION_DISCOVERY_FINISHED); this.registerReceiver(mReceiver, filter); - // Get the local Bluetooth adapter - mBtAdapter = BluetoothAdapter.getDefaultAdapter(); - // Get a set of currently paired devices Set pairedDevices = mBtAdapter.getBondedDevices(); @@ -111,14 +135,26 @@ public void onClick(View v) { if (pairedDevices.size() > 0) { findViewById(R.id.title_paired_devices).setVisibility(View.VISIBLE); for (BluetoothDevice device : pairedDevices) { - mPairedDevicesArrayAdapter.add(device.getName() + "\n" + device.getAddress()); + pairedDevicesArrayAdapter.add(device.getName() + "\n" + device.getAddress()); } } else { - String noDevices = getResources().getText(R.string.none_paired).toString(); - mPairedDevicesArrayAdapter.add(noDevices); + String noDevices = getResources().getText(R.string.bluetooth_none_paired).toString(); + pairedDevicesArrayAdapter.add(noDevices); + } + } + + public void requestBluetooth(){ + if (!mBtAdapter.isEnabled()) { + Intent enableBtIntent = new Intent(BluetoothAdapter.ACTION_REQUEST_ENABLE); + startActivityForResult(enableBtIntent, REQUEST_ENABLE_BT); } } + public void blueToothIsNotSupported(){ + Toast.makeText(this, R.string.bluetooth_not_supported, Toast.LENGTH_SHORT).show(); + finish(); + } + @Override protected void onDestroy() { super.onDestroy(); @@ -132,11 +168,33 @@ protected void onDestroy() { this.unregisterReceiver(mReceiver); } + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + + switch (requestCode) { + + case REQUEST_ENABLE_BT: + if (resultCode != Activity.RESULT_OK || resultCode == Activity.RESULT_CANCELED) { + Toast.makeText(this, R.string.bluetooth_not_enabled_leaving, Toast.LENGTH_SHORT).show(); + finish(); + } else { + // Can use bluetooth + } + break; + } + } + /** * Start device discover with the BluetoothAdapter */ private void doDiscovery() { - if (D) Log.d(TAG, "doDiscovery()"); + Log.d(TAG, "doDiscovery()"); + + if (mBtAdapter == null) { + finish(); + return; + } // Indicate scanning in the title setProgressBarIndeterminateVisibility(true); @@ -154,8 +212,11 @@ private void doDiscovery() { mBtAdapter.startDiscovery(); } - // The on-click listener for all devices in the ListViews - private OnItemClickListener mDeviceClickListener = new OnItemClickListener() { + /** + * The on-click listener for all devices in the ListViews + */ + private AdapterView.OnItemClickListener mDeviceClickListener + = new AdapterView.OnItemClickListener() { public void onItemClick(AdapterView av, View v, int arg2, long arg3) { // Cancel discovery because it's costly and we're about to connect mBtAdapter.cancelDiscovery(); @@ -174,8 +235,10 @@ public void onItemClick(AdapterView av, View v, int arg2, long arg3) { } }; - // The BroadcastReceiver that listens for discovered devices and - // changes the title when discovery is finished + /** + * The BroadcastReceiver that listens for discovered devices and changes the title when + * discovery is finished + */ private final BroadcastReceiver mReceiver = new BroadcastReceiver() { @Override public void onReceive(Context context, Intent intent) { @@ -189,16 +252,16 @@ public void onReceive(Context context, Intent intent) { if (device.getBondState() != BluetoothDevice.BOND_BONDED) { mNewDevicesArrayAdapter.add(device.getName() + "\n" + device.getAddress()); } - // When discovery is finished, change the Activity title + // When discovery is finished, change the Activity title } else if (BluetoothAdapter.ACTION_DISCOVERY_FINISHED.equals(action)) { setProgressBarIndeterminateVisibility(false); setTitle(R.string.select_device); if (mNewDevicesArrayAdapter.getCount() == 0) { - String noDevices = getResources().getText(R.string.none_found).toString(); + String noDevices = getResources().getText(R.string.bluetooth_none_found).toString(); mNewDevicesArrayAdapter.add(noDevices); } } } }; -} +} \ No newline at end of file diff --git a/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/Lexicon.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/Lexicon.java new file mode 100644 index 0000000..ebad7c5 --- /dev/null +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/Lexicon.java @@ -0,0 +1,204 @@ +package com.androidmontreal.gesturevoicecommander.robots; + +import android.content.Intent; + +import java.util.ArrayList; +import java.util.Locale; + +public class Lexicon { + private int language; + private int timer = 5; + + /* + * Languages + */ + public static final int EN = 1000; + public static final int FR = 1001; + public static final String EN_CARRIER_PHRASE = "I will tell the robot: "; + public static final String FR_CARRIER_PHRASE = "Je vais dire au robo: "; + + /* + * Commands, use the order of these constants for the precedence order of the + * commands (STOP has the highest precedence) + */ + public static final int STOP = 0; + public static final int EXPLORE = 1; + public static final int FORWARD = 2; + public static final int REVERSE = 3; + public static final int ROTATERIGHT = 4; + public static final int ROTATELEFT = 5; + public static final int TURNRIGHT = 6; + public static final int TURNLEFT = 7; + public static final int CONNECT = 8; + + private ArrayList en; + private ArrayList fr; + + private int mCommandToExecute; + + public String stop() { + return "S"; + } + + public String explore() { + return "E"; + } + + public String forward() { + return "1F2F"; + } + + public String reverse() { + return "1R2R"; + } + + public String turnRight() { + return "1F"; + } + + public String turnLeft() { + return "2F"; + } + + public String rotateRight() { + return "1F2R"; + } + + public String rotateLeft() { + return "1R2F"; + } + + public String connect() { + return "CONNECT"; + } + + public void defineLanguages() { + en = new ArrayList(); + en.add(STOP, "stop:wait:don't:no:damn"); + en.add(EXPLORE, "explore:try"); + en.add(FORWARD, "forward:ahead"); + en.add(REVERSE, "reverse:back"); + en.add(ROTATERIGHT, "rotate&right"); + en.add(ROTATELEFT, "rotate&left"); + en.add(TURNRIGHT, "right"); + en.add(TURNLEFT, "left"); + en.add(CONNECT, "connect:body:robot:bluetooth"); + + fr = new ArrayList(); + fr.add(STOP, "arrête:arrêter:arrêté:arrêter:arrêtez:pas:voyons:voyant:m****:merde:zut:non:stop"); + fr.add(EXPLORE, "explore:explorer"); + fr.add(FORWARD, "avance:tu dois:tu dois:te doi:tournoi:tout droit:avanc:forward"); + fr.add(REVERSE, "recule:recul:reverse:arrière"); + fr.add(ROTATERIGHT, "pivoter vers la droite:vers la droite:pilot:pivot:rotate&droit"); + fr.add(ROTATELEFT, "pivoter vers la gauche:vers la gauche:rotate&gauche"); + fr.add(TURNRIGHT, "à la droite:droite:droit:right"); + fr.add(TURNLEFT, "à la gauche:gauche:left"); + fr.add(CONNECT, "connecter:mon corps:robot:bluetooth"); + } + + public String execute(int commandInteger) { + switch (commandInteger) { + case CONNECT: + return connect(); + case STOP: + return stop(); + case EXPLORE: + return explore(); + case FORWARD: + return forward(); + case REVERSE: + return reverse(); + case TURNRIGHT: + return turnRight(); + case TURNLEFT: + return turnLeft(); + case ROTATERIGHT: + return rotateRight(); + case ROTATELEFT: + return rotateLeft(); + default: + return stop(); + } + } + + public String guessWhatToDo(String command) { + command = command.toLowerCase(); + int commandToExecute = STOP; + String commandForHumans = command; + + ArrayList humancommands = en; + if (language == FR) { + humancommands = fr; + } + for (int i = 0; i < humancommands.size(); i++) { + String[] andwords = humancommands.get(i).split("&"); + String[] orwords = humancommands.get(i).split(":"); + /* + * If there are AND words, then check first to see if it matches all words + */ + if (andwords.length > 1) { + int wordsfound = 0; + commandForHumans = andwords[0]; + for (int k = 0; k < andwords.length; k++) { + if (command.contains(andwords[k])) { + wordsfound++; + } + } + if (wordsfound >= andwords.length) { + commandToExecute = i; + mCommandToExecute = commandToExecute; + return commandForHumans; + } + } + /* + * Then if a command hasn't been issued, check for the OR words. + */ + if (orwords.length > 0) { + commandForHumans = orwords[0]; + for (int k = 0; k < orwords.length; k++) { + if (command.contains(orwords[k])) { + commandToExecute = i; + mCommandToExecute = commandToExecute; + return commandForHumans; + } + } + } + + } + mCommandToExecute = commandToExecute; + return commandForHumans; + } + + public String executeGuess() { + if (mCommandToExecute >= 0) { + return execute(mCommandToExecute); + } + return ""; + } + + public Lexicon(int language, int timer) { + super(); + defineLanguages(); + this.language = language; + this.timer = timer; + } + + public Lexicon(int language) { + super(); + defineLanguages(); + this.language = language; + this.timer = 5; + } + + public Lexicon() { + super(); + defineLanguages(); + if (Locale.getDefault().getLanguage().contains("fr")) { + this.language = FR; + } else { + this.language = EN; + } + this.timer = 5; + } + +} diff --git a/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/Robot.java b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/Robot.java new file mode 100644 index 0000000..73bf2dd --- /dev/null +++ b/Tutorial/app/src/main/java/com/androidmontreal/gesturevoicecommander/robots/Robot.java @@ -0,0 +1,341 @@ +package com.androidmontreal.gesturevoicecommander.robots; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; + +import android.app.Activity; +import android.bluetooth.BluetoothAdapter; +import android.bluetooth.BluetoothDevice; +import android.content.Intent; +import android.content.pm.PackageManager; +import android.content.pm.ResolveInfo; +import android.gesture.Gesture; +import android.gesture.GestureLibraries; +import android.gesture.GestureLibrary; +import android.gesture.GestureOverlayView; +import android.gesture.Prediction; +import android.gesture.GestureOverlayView.OnGesturePerformedListener; +import android.os.Bundle; +import android.os.Handler; +import android.os.Message; +import android.speech.RecognizerIntent; +import android.speech.tts.TextToSpeech; +import android.speech.tts.TextToSpeech.OnInitListener; +import android.util.Log; +import android.view.View; +import android.widget.Toast; + +import com.androidmontreal.gesturevoicecommander.GestureBuilderActivity; +import com.androidmontreal.gesturevoicecommander.R; + +/** + * Building on what we saw in MakeItUnderstand, now lets make it perform + * actions. Here is some super simple code that builds on the BluetoothChat + * sample code to send meassages to a bluetooth device/robot. + * + * @author cesine + */ +public class Robot extends Activity implements OnInitListener, OnGesturePerformedListener { + private static final String TAG = "Robot"; + private static final int RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE = 341; + public static final int REQUEST_CONNECT_DEVICE = 8888; + + private static final boolean D = true; + + /** + * Talk to the user + */ + private TextToSpeech mTts; + + /* + * A gesture library we created with the GestureBuilder, saved on the SDCard + * and then imported into the res/raw folder of this project + */ + private GestureLibrary gestureLib; + + /* A little lexicon we made for the DFR Rover at Cloud Robotics Hackathon */ + private Lexicon lexicon; + + /* A re-executable sequence of commands in time */ + private HashMap mCommandMemory; + + /* Message passing to an actual bluetooth device/robot */ + private BluetoothAdapter mBluetoothAdapter = null; + private BluetoothChatService mChatService; + private String mConnectedDeviceName = ""; + private static final boolean SECURE_CONNECTION = true; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + mTts = new TextToSpeech(this, this); + + GestureOverlayView gestureOverlayView = new GestureOverlayView(this); + View inflate = getLayoutInflater().inflate(R.layout.commander, null); + gestureOverlayView.addView(inflate); + gestureOverlayView.addOnGesturePerformedListener(this); + // gestureLib = GestureLibraries.fromFile(fileOnYourSDCard); + gestureLib = GestureLibraries.fromRawResource(this, R.raw.gestures); + if (!gestureLib.load()) { + Toast.makeText(this, R.string.gestures_empty, Toast.LENGTH_SHORT).show(); + finish(); + } + setContentView(gestureOverlayView); + + lexicon = new Lexicon(); + } + + @Override + protected void onResume() { + super.onResume(); + + if (mCommandMemory == null) { + mCommandMemory = new HashMap(); + } + + if (mChatService != null) { + // Only if the state is STATE_NONE, do we know that we haven't started already + if (mChatService.getState() == BluetoothChatService.STATE_NONE) { + // Start the Bluetooth chat services + mChatService.start(); + } + } + } + + protected void promptTheUserToTalk() { + if (isIntentAvailable(RecognizerIntent.ACTION_RECOGNIZE_SPEECH)) { + this.speak(getString(R.string.im_listening)); + } else { + this.speak(getString(R.string.i_cant_listen)); + } + } + + /** + * Fire an intent to start the voice recognition activity. + */ + private void startVoiceRecognitionActivity() { + Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); + intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM); + intent.putExtra(RecognizerIntent.EXTRA_PROMPT, getString(R.string.im_listening)); + if (isIntentAvailable(intent)) { + startActivityForResult(intent, RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE); + } else { + Log.w(TAG, "This device doesn't have speech recognition, maybe its an emulator or a phone from china without google products?"); + } + } + + /** + * Handle the results from the voice recognition activity. + */ + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + + switch (requestCode) { + case RETURN_FROM_VOICE_RECOGNITION_REQUEST_CODE: + if (resultCode == RESULT_OK) { + ArrayList matches = data.getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS); + /* try to find a robot command in the first match */ + if (matches.size() > 0) { + sendRobotThisCommand(matches.get(0)); + } + } + break; + case REQUEST_CONNECT_DEVICE: + // When DeviceListActivity returns with a device to connect + if (resultCode == Activity.RESULT_OK) { + if (mChatService == null) { + mChatService = new BluetoothChatService(this, mHandler); + } + connectDevice(data); + } + break; + } + } + + @Override + protected void onDestroy() { + if (mTts != null) { + mTts.stop(); + mTts.shutdown(); + } + + if (mChatService != null) { + mChatService.stop(); + } + + super.onDestroy(); + } + + @Override + public void onInit(int status) { + if (status == TextToSpeech.SUCCESS) { + int result = mTts.setLanguage(Locale.getDefault()); + if (result == TextToSpeech.LANG_MISSING_DATA || result == TextToSpeech.LANG_NOT_SUPPORTED) { + Log.e(TAG, "Language is not available."); + Toast.makeText(this, + "The " + Locale.getDefault().getDisplayLanguage() + + " TextToSpeech isn't installed, you can go into the " + + "\nAndroid's settings in the " + + "\nVoice Input and Output menu to turn it on. ", + Toast.LENGTH_LONG).show(); + } else { + // everything is working. + this.speak(getString(R.string.instructions_to_look_at_menu)); + } + } else { + Toast.makeText(this, "Sorry, I can't talk to you because " + + "I could not initialize TextToSpeech.", Toast.LENGTH_LONG).show(); + } + } + + public boolean speak(String message) { + if (mTts != null) { + mTts.speak(message, TextToSpeech.QUEUE_ADD, null); + } else { + Toast.makeText(this, "Sorry, I can't speak to you: " + message, Toast.LENGTH_LONG).show(); + } + return true; + } + + @Override + public void onGesturePerformed(GestureOverlayView overlay, Gesture gesture) { + ArrayList predictions = gestureLib.recognize(gesture); + for (Prediction prediction : predictions) { + if (prediction.score > 3.0) { + Log.d(TAG, "Detected this gesture " + prediction.name + " with a score of " + prediction.score); + } + } + if (predictions.size() > 0) { + sendRobotThisCommand(predictions.get(0).name); + } + } + + /** + * Establish connection with a physical device/body via bluetooth + * + * @param data An {@link Intent} with {@link DeviceListActivity#EXTRA_DEVICE_ADDRESS} extra. + */ + private void connectDevice(Intent data) { + // Get the device MAC address + String address = data.getExtras().getString(DeviceListActivity.EXTRA_DEVICE_ADDRESS); + // Get the BluetoothDevice object + if (mBluetoothAdapter == null) { + mBluetoothAdapter = BluetoothAdapter.getDefaultAdapter(); + } + BluetoothDevice device = mBluetoothAdapter.getRemoteDevice(address); + // Attempt to connect to the device + mChatService.connect(device, SECURE_CONNECTION); + } + + public String sendRobotThisCommand(String requestedCommand) { + String understoodCommand = lexicon.guessWhatToDo(requestedCommand); + + // communicate understood command + Toast.makeText(this, understoodCommand, Toast.LENGTH_SHORT).show(); + if (Locale.getDefault().getLanguage().contains("fr")) { + mTts.speak(lexicon.FR_CARRIER_PHRASE + understoodCommand, TextToSpeech.QUEUE_ADD, null); + } else { + mTts.speak(lexicon.EN_CARRIER_PHRASE + understoodCommand, TextToSpeech.QUEUE_ADD, null); + } + + // remember understood command + mCommandMemory.put(System.currentTimeMillis(), "I want to: " + understoodCommand); + + // translate into body commands + String bodyCommand = lexicon.executeGuess(); + if ("CONNECT".equals(bodyCommand)) { + Intent serverIntent = new Intent(this, DeviceListActivity.class); + startActivityForResult(serverIntent, REQUEST_CONNECT_DEVICE); + return ""; + } + if (mChatService != null){ + // Check that we're actually connected before trying anything + if (mChatService.getState() != BluetoothChatService.STATE_CONNECTED) { + Toast.makeText(this, R.string.bluetooth_not_connected, Toast.LENGTH_SHORT).show(); + return understoodCommand; + } + // Check that there's actually something to send + if (bodyCommand.length() > 0) { + // Get the message bytes and tell the BluetoothChatService to write + byte[] send = bodyCommand.getBytes(); + mChatService.write(send); + } + } + + return understoodCommand; + } + + /** + * The Handler that gets information back from the BluetoothChatService + */ + private final Handler mHandler = new Handler() { + @Override + public void handleMessage(Message msg) { + switch (msg.what) { + case BluetoothChatService.MESSAGE_STATE_CHANGE: + switch (msg.arg1) { + case BluetoothChatService.STATE_CONNECTED: + Log.d(TAG, getString(R.string.title_connected_to, mConnectedDeviceName)); + break; + case BluetoothChatService.STATE_CONNECTING: + Log.d(TAG, "title_connecting"); + break; + case BluetoothChatService.STATE_LISTEN: + break; + case BluetoothChatService.STATE_NONE: + Log.d(TAG, "title_not_connected"); + break; + } + break; + case BluetoothChatService.MESSAGE_WRITE: + byte[] writeBuf = (byte[]) msg.obj; + // construct a string from the buffer + String bodyCommand = new String(writeBuf); + mCommandMemory.put(System.currentTimeMillis(), "I told my body to: " + bodyCommand); + break; + case BluetoothChatService.MESSAGE_READ: + byte[] readBuf = (byte[]) msg.obj; + // construct a string from the valid bytes in the buffer + String readMessage = new String(readBuf, 0, msg.arg1); + speak(readMessage); + mCommandMemory.put(System.currentTimeMillis(), "My body did: " + readMessage); + break; + case BluetoothChatService.MESSAGE_DEVICE_NAME: + // save the connected device's name + mConnectedDeviceName = msg.getData().getString(BluetoothChatService.DEVICE_NAME); + Toast.makeText(getApplicationContext(), "My body is: " + + mConnectedDeviceName, Toast.LENGTH_SHORT).show(); + break; + case BluetoothChatService.MESSAGE_TOAST: + Toast.makeText(getApplicationContext(), msg.getData().getString(BluetoothChatService.TOAST), + Toast.LENGTH_SHORT).show(); + break; + } + } + }; + + public void onCommandByVoiceClick(View v) { + promptTheUserToTalk(); + startVoiceRecognitionActivity(); + } + + public void onViewGesturesClick(View v) { + Intent i = new Intent(this, GestureBuilderActivity.class); + startActivity(i); + } + + public boolean isIntentAvailable(String action) { + final Intent intent = new Intent(action); + return isIntentAvailable(intent); + } + + public boolean isIntentAvailable(final Intent intent) { + final PackageManager packageManager = this.getPackageManager(); + List list = packageManager.queryIntentActivities(intent, PackageManager.MATCH_DEFAULT_ONLY); + return list.size() > 0; + } +} diff --git a/GestureVoiceCommander/res/drawable-hdpi/app_icon.png b/Tutorial/app/src/main/res/drawable-hdpi/ic_launcher.png similarity index 99% rename from GestureVoiceCommander/res/drawable-hdpi/app_icon.png rename to Tutorial/app/src/main/res/drawable-hdpi/ic_launcher.png index c609872..d62b72a 100644 Binary files a/GestureVoiceCommander/res/drawable-hdpi/app_icon.png and b/Tutorial/app/src/main/res/drawable-hdpi/ic_launcher.png differ diff --git a/GestureVoiceCommander/res/drawable-ldpi/ic_launcher.png b/Tutorial/app/src/main/res/drawable-ldpi/ic_launcher.png similarity index 100% rename from GestureVoiceCommander/res/drawable-ldpi/ic_launcher.png rename to Tutorial/app/src/main/res/drawable-ldpi/ic_launcher.png diff --git a/GestureVoiceCommander/res/drawable-mdpi/ic_launcher.png b/Tutorial/app/src/main/res/drawable-mdpi/ic_launcher.png similarity index 100% rename from GestureVoiceCommander/res/drawable-mdpi/ic_launcher.png rename to Tutorial/app/src/main/res/drawable-mdpi/ic_launcher.png diff --git a/AndroidArduinoOpenCV/res/layout/device_list.xml b/Tutorial/app/src/main/res/layout/activity_device_list.xml similarity index 82% rename from AndroidArduinoOpenCV/res/layout/device_list.xml rename to Tutorial/app/src/main/res/layout/activity_device_list.xml index 395695f..576f7fa 100644 --- a/AndroidArduinoOpenCV/res/layout/device_list.xml +++ b/Tutorial/app/src/main/res/layout/activity_device_list.xml @@ -1,5 +1,5 @@ - - - + + - + + - + + -