c# - This code hits a roadblock on Android, any idea why? -


the method below runs great on ios iphone 5, mac, pc, , web has major slowdown on android. in unity 5 c#.

using unityengine; using unityengine.ui; using system.collections; using opencvforunity;  public class mouthdetect : monobehaviour {      private webcamtexture _webcamtexture;     private color32[] _colors;     public bool isfrontfacing = true;     public int webcamtexturewidth = 640;     public int webcamtextureheight = 480;     public int webcamtexturefps = 10;     private string _firstcam; // debuging no front facing camera.     private mat _rgbamat;     private mat _graymat;     private mat _mouthmatgray;     private mat _mouthmatrgba;     private texture2d _texture;     private cascadeclassifier _cascadeface;     private cascadeclassifier _cascademouth;     private matofrect _faces;     private matofrect _mouths;     private bool _initdone = false;     private opencvforunity.rect _mouthroi = new opencvforunity.rect();      // chew detection     private unityengine.rect _rectmouth;     private int _mouthwidth;  //      private int _mouthheight;     private double _totalbright = new double();  // total brightness inside mouth     private double _lastbright = 0; // last extreme value of open or closed.     private texture2d _textmouth;     private bool _mouthisopenboolean = false;  // mouth open? boolean     public double mouthchangediff = 0.2;  // difference between brightness before mout opened or closed.  smaller detects more chews can prone false chews.     public int mouthdetectpixelcount = 10;  // how many pixels should use detection?     private bool _mouthfound = false;     private bool _facefound = false;      // debug logic     private gameobject _previewtoggle;     private gameobject _previewscreen;     private image _previewimage;     private facedetpreview _previewscript;     private bool _debugpreviewon = false;     private sprite _webcamsprite;     private string _camerainf = "";     private gameobject _cameralist;     private debugcameras _cameralistscript;     private string _selectedcamera = "";      // create array of opencv rectangles array of faces. andy     opencvforunity.rect[] rects;     // create array of opencv rectangles array of mouths.     opencvforunity.rect[] rectsm;      // detection logic     public int logicsolution = 1; // solution using?  0 = brightness, 1 = find/lose mouth.     private bool _capturingface = false;      public bool mouthisopenboolean {         { return _mouthisopenboolean; }         set {              // make sure we've changed.             if (_mouthisopenboolean==value)                  return;              _mouthisopenboolean = value;               // add chew if we've changed false             if (value == false) {                 gameobject chewtext = gameobject.find ("chews");                 chewing chewing = chewtext.getcomponent<chewing>();                 chewing.addchew();             }         }     }       // use initialization     void start ()     {         // debug preview toggle , image         _previewtoggle = gameobject.find ("facepreviewtoggle");         _previewscript = _previewtoggle.getcomponent<facedetpreview> ();         _previewscreen =  gameobject.find ("facepreview");         _previewimage = _previewscreen.getcomponent<image> ();         _cameralist = gameobject.find ("cameras");         if(_cameralist!=null)             _cameralistscript = _cameralist.getcomponent<debugcameras>();          // setup camera , texture         startcoroutine (init ());     }      private ienumerator init ()     {         // have initialised?  restart if have.         if (_webcamtexture != null) {             _webcamtexture.stop ();             _initdone = false;              _rgbamat.dispose ();             _graymat.dispose ();         }          // checks how many , cameras available on device         _camerainf = "cameras=" + webcamtexture.devices.length.tostring ();         (int cameraindex = 0; cameraindex < webcamtexture.devices.length; cameraindex++) {             // debug information             addtodebug( "camera " + cameraindex.tostring() + " - " + webcamtexture.devices [cameraindex].name +                         " - front:" + webcamtexture.devices [cameraindex].isfrontfacing);              // want using correct camera, make sure it's front facing             if (webcamtexture.devices [cameraindex].isfrontfacing == isfrontfacing) {                 // tell camera we're using                 debug.log (cameraindex + " name " + webcamtexture.devices [cameraindex].name + " isfrontfacing " + webcamtexture.devices [cameraindex].isfrontfacing);                 debug.log("here");                  // set camera texture                 _webcamtexture = new webcamtexture (webcamtexture.devices [cameraindex].name, webcamtexturewidth, webcamtextureheight);                 break;             }             else {                  // want first 1                 if (_firstcam=="") {                     _firstcam=webcamtexture.devices [cameraindex].name;                  }             }         }          // did find camera?         if (_webcamtexture == null) {             // if we're in unity ok...             #if unity_editor             // create new camera             _webcamtexture = new webcamtexture (_firstcam, webcamtexturewidth, webcamtextureheight);             #endif              // ****** need stop here!!!!!! ***************************************************************************             // todo: device has no forward facing camera             // can't chew without front facing camera!!!         }          // debug...         addtodebug ("selected camera: " + _webcamtexture.devicename);         debug.log(_webcamtexture.devicename);          // how many fps want??  need control performance.  although, request ignored! :/         _webcamtexture.requestedfps = webcamtexturefps;          // start camera         _webcamtexture.play ();          while (true) {             //if want use webcamtexture.width , webcamtexture.height on ios, have wait until webcamtexture.didupdatethisframe == 1, otherwise these 2 values equal 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/)             if (_webcamtexture.width > 16 && _webcamtexture.height > 16) {                 // create new colour , matrices                 _colors = new color32[_webcamtexture.width * _webcamtexture.height];                 _rgbamat = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc4);                 _graymat = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc1);                 _mouthmatrgba = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc4);                 _mouthmatgray = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc1);                 _texture = new texture2d (_webcamtexture.width, _webcamtexture.height, textureformat.rgba32, false);                 _textmouth = new texture2d (_webcamtexture.width, _webcamtexture.height, textureformat.rgba32, false);                  // set rotation                 //              gameobject.transform.eulerangles = new vector3 (0, 0, 0);                  // android , iphone rotation needs change correct z axis                 /*              #if (unity_android || unity_iphone) && !unity_editor                     gameobject.transform.eulerangles = new vector3 (0, 0, -90);                 #endif */                 // i'm not sure why removed in sample, i'll leave out until figure out it's ueseful for,                   // looks developer trying dynamically correct rotated webcams.                 //  gameobject.transform.rotation = gameobject.transform.rotation * quaternion.angleaxis (_webcamtexture.videorotationangle, vector3.back);                  // looks dynamically fixing scale????  why dev removing this???  may re-introduce it.                 /*  bool videoverticallymirrored = _webcamtexture.videoverticallymirrored;                     float scalex = 1;                     float scaley = videoverticallymirrored ? -1.0f : 1.0f;                     if (_webcamtexture.videorotationangle == 270)                         scaley = -1.0f;                     gameobject.transform.localscale = new vector3 (scalex * gameobject.transform.localscale.x, scaley * gameobject.transform.localscale.y, 1);                 */                  // create new classifiers - todo have work path make work android , iphone...                 _cascadeface = new cascadeclassifier (utils.getfilepath ("haarcascade_frontalface_alt.xml"));                  _cascademouth = new cascadeclassifier (utils.getfilepath ("haarcascade_mouth.xml"));                     // create matrix of faces (we'll load @ point???....)                 _faces = new matofrect ();                 _mouths = new matofrect ();                  // set camera ortho size, don't this work frog camera.                 camera.main.orthographicsize = _webcamtexture.width / 2;                  // we're initilised                 _initdone = true;                  break;             }              else {                 // come , try again, we're not ready yet.                 yield return 0;             }         }     }      // update called once per frame     void update ()     {         // make sure we've initialised.         if ((_initdone) && (!_capturingface)) {             startcoroutine ("captureface");         }     }      private ienumerator captureface() {         _capturingface = true;          // make sure texture has been correctly formated, if not we'll have come later.         if (_webcamtexture.width > 16 && _webcamtexture.height > 16) {              // pass web cam texture opencv matrix             utils.webcamtexturetomat (_webcamtexture, _rgbamat, _colors);              // iphones buggering mirroring again...             #if unity_iphone && !unity_editor             // flip if neccessary             if (_webcamtexture.videoverticallymirrored){                 if(isfrontfacing){                     core.flip (_rgbamat, _rgbamat, 1);                 }else{                     core.flip (_rgbamat, _rgbamat, 0);                 }             }else{                 if(isfrontfacing){                     core.flip (_rgbamat, _rgbamat, -1);                 }             }             #endif              // convert rgb web texture matrix gray             imgproc.cvtcolor (_rgbamat, _graymat, imgproc.color_rgba2gray);              // adjust contrast - can impact performance, try without faster performance             imgproc.equalizehist (_graymat, _graymat);              // set cascade detect different sized targets             if (_cascadeface != null)                 _cascadeface.detectmultiscale (_graymat, _faces, 1.1, 2, 2, // todo: objdetect.cv_haar_scale_image                                                new size (_webcamtexture.width * 0.15, _webcamtexture.width * 0.15), new size ());              // create array of opencv rectangles array of faces.             rects = _faces.toarray ();              // find mouth in each face each face.             _facefound = false;             (int = 0; < rects.length; i++) {                  // found face                 _facefound = true;                  // debugging, show face is.                 core.rectangle (                     _rgbamat,                      new point (rects [i].x ,rects [i].y),                      new point (rects [i].x + rects [i].width, rects [i].y + rects [i].height),                      new scalar (0, 255, 0, 255),                      2);                  // create rectangle around region of face we're interested in                 // keep inside face box, otherwise errors when we're @ edge of screen                 /*              opencvforunity.rect _mouthroi = new opencvforunity.rect(                     (int)rects [i].x,                     (int)rects [i].y + (rects [i].height / 2),                     (int)rects [i].width,                     (int)(rects [i].height- (rects [i].height / 2))); */              opencvforunity.rect _mouthroi = new opencvforunity.rect(            // improved sensitivity!!!!                                                                              (int)rects [i].x,                                                                              (int)rects [i].y + ((rects [i].height / 3) * 2),                                                                              (int)rects [i].width,                                                                              (int)(rects [i].height - ((rects [i].height / 3) * 2)));                  // create new matrix using roi                  _mouthmatgray = _graymat.submat(_mouthroi);                 _mouthmatrgba = _rgbamat.submat(_mouthroi);                  // detect mouth (we're going use first mouth captured                 // set cascade detect different sized targets                 if (_cascademouth != null)                     _cascademouth.detectmultiscale (_mouthmatgray, _mouths, 1.1, 2, 2, // todo: objdetect.cv_haar_scale_image                                                     new size (_webcamtexture.width * 0.04, _webcamtexture.width * 0.04), new size ());                 // create array of opencv rectangles array of mouths.                 rectsm = _mouths.toarray ();                  // put rectangle around first mouth on each face                 _mouthfound = false;                 (int j = 0; j < rectsm.length; j++) {                     // set mouth box , make sure x , y correct (remember, these co-ords inside roi)                     _rectmouth = new unityengine.rect(                         rectsm[j].x + _mouthroi.x,                         _webcamtexture.height - _mouthroi.y - rectsm[j].y - rectsm[j].height,                         rectsm[j].width,                         rectsm[j].height                         );                      // we've found mouth!                      _mouthfound = true;                      core.rectangle (                         _rgbamat,                          new point (rectsm [j].x + _mouthroi.x,rectsm [j].y + _mouthroi.y),                          new point (rectsm [j].x + rectsm [j].width + _mouthroi.x, rectsm [j].y + rectsm [j].height + _mouthroi.y),                          new scalar (0, 0, 255, 255),                          2);                     break;                 }             }              // add output asssigned texture!             utils.mattotexture2d (_rgbamat, _texture, _colors);              // debugging?             _debugpreviewon = _previewscript.istoggled;             if (_debugpreviewon) {                 _webcamsprite = sprite.create(                     _texture,                      new unityengine.rect(                     0,                     0,                     _texture.width,                     _texture.height),                      new vector2(0,0));                 _previewimage.sprite = _webcamsprite;             }              //debug.log("face:" + _facefound.tostring() + " mouth:" + _mouthfound.tostring());                // version using??  fact have mouth satisfy "find/lose mouth" method             if (logicsolution==1) {                 // we're setting oposite of mouth found (ie, if we've found mouth it's closed                 mouthisopenboolean = !_mouthfound;             }              // continue solution 0 , have mouth.             if ((logicsolution==0) && (_mouthfound==true)) {                 debug.log("we shouldn't see this");                  // how high , wide mouth                 _mouthwidth = (int)_rectmouth.width;                 _mouthheight = (int)_rectmouth.height;                  // create mouth texture                 color[] pixels = _texture.getpixels(                     (int)_rectmouth.x,                     (int)_rectmouth.y,                     (int)_rectmouth.width,                     (int)_rectmouth.height);                 _textmouth.resize(_mouthwidth, _mouthheight);                 _textmouth.apply();                 _textmouth.setpixels(pixels);                 _textmouth.apply();                  // work out detection pixels                 int mouthcentre = _mouthwidth / 2;                 _totalbright = 0;                 int mouthgap = _mouthheight / mouthdetectpixelcount;                 unityengine.rect rectshapepreview =                      new unityengine.rect((float)mouthcentre - 1, (float)mouthgap,3, (float)mouthgap*(float)mouthdetectpixelcount);                  // work out gap between pixels                 (int = 1; <= mouthdetectpixelcount; i++)                 {                     // work out total brightness                     int pixely = mouthgap * i;                     _totalbright += brightness(_textmouth.getpixel(mouthcentre, pixely));                 }                  // have opened or closed.                 //                 // rules (for boolean):                 //  1. initial change 2 tenths in eather direction (double mouthchangediff) change open or closed.                   //      smaller = open, bigger = closed.                 //  2. record current number _lastbright (double) , start closed (bool - false) in mouthisopenboolean                 //  3. if number gets bigger keep updating _lastbright variable make note of "close" brightness.                 //  4. if number gets smaller measure it, if it's more 2/10s (or ever variable we're open:                 //      change bool true , record number.                 //      if it's less leave it, if capture mouth when it's mid open or mid close.                 //  5. if we've moved closed stops to 4 in oposite direction                 //                 // rules (for integer)                 //  1. use boolean logic change diff maximimum 100 (open - oposite boolean :s) , minimum 0 (closed);                 //  2. use increment tie setting nearest increment.                  // record current number _lastbright if first capture                 if (_lastbright == 0) {                     _lastbright = _totalbright;                 }                  else {                     //get difference                     double diff = new double();                     diff = _lastbright - _totalbright;                      // need positive diff                     double posdiff = diff;                     if (posdiff < 0) {posdiff = -posdiff;}                      // have gone more open or more closed?                     if (((_lastbright > _totalbright) && (mouthisopenboolean)) ||                         ((_lastbright < _totalbright) && (!mouthisopenboolean))) {                          // we're going further in same direction, log last brighness                         _lastbright = _totalbright;                     }                     else {                         // we've changed direction.  if more diff level set???                         if (posdiff > mouthchangediff) {                             // we've changed direction!                             if (mouthisopenboolean) {                                 mouthisopenboolean = false;                             }                             else {                                 mouthisopenboolean = true;                             }                              // set new last brightness                             _lastbright = _totalbright;                         }                        }                 }             }         }          yield return new waitforseconds (0.1f);     // massive performance boost.         _capturingface = false;         yield return null;     }      void ondisable ()     {         // stop webcam.         _webcamtexture.stop ();     }      void ongui ()     {         float screenscale = screen.width / 240.0f;         matrix4x4 scaledmatrix = matrix4x4.scale (new vector3 (screenscale, screenscale, screenscale));         gui.matrix = scaledmatrix;          // how big our gui box?         //      unityengine.rect devrect = new unityengine.rect (1, 1, (_texture.width/10)+2, (_texture.height/10)+90);      }      // return brighness of colour of pixel     private double brightness(color c)      {         return mathf.sqrt (             c.r * c.r * .241f +              c.g * c.g * .691f +              c.b * c.b * .068f);     }      public void addtodebug(string newline) {          // need add carraige return         if (_camerainf != "")              _camerainf += "\n";          // add line , update         _camerainf += newline;         if (_cameralistscript!=null)             _cameralistscript.cameralist = _camerainf;      }  } 

specifically in update() ienumerator captureface() taking 150ms plus run according profiler.

because use cpu-heavy functions,

_textmouth.setpixels(pixels); _textmouth.apply(); 

and it's not platform-dependent issue.


Comments

Popular posts from this blog

Email notification in google apps script -

c++ - Difference between pre and post decrement in recursive function argument -

javascript - IE11 incompatibility with jQuery's 'readonly'? -