c# - This code hits a roadblock on Android, any idea why? -
the method below runs great on ios iphone 5, mac, pc, , web has major slowdown on android. in unity 5 c#.
using unityengine; using unityengine.ui; using system.collections; using opencvforunity; public class mouthdetect : monobehaviour { private webcamtexture _webcamtexture; private color32[] _colors; public bool isfrontfacing = true; public int webcamtexturewidth = 640; public int webcamtextureheight = 480; public int webcamtexturefps = 10; private string _firstcam; // debuging no front facing camera. private mat _rgbamat; private mat _graymat; private mat _mouthmatgray; private mat _mouthmatrgba; private texture2d _texture; private cascadeclassifier _cascadeface; private cascadeclassifier _cascademouth; private matofrect _faces; private matofrect _mouths; private bool _initdone = false; private opencvforunity.rect _mouthroi = new opencvforunity.rect(); // chew detection private unityengine.rect _rectmouth; private int _mouthwidth; // private int _mouthheight; private double _totalbright = new double(); // total brightness inside mouth private double _lastbright = 0; // last extreme value of open or closed. private texture2d _textmouth; private bool _mouthisopenboolean = false; // mouth open? boolean public double mouthchangediff = 0.2; // difference between brightness before mout opened or closed. smaller detects more chews can prone false chews. public int mouthdetectpixelcount = 10; // how many pixels should use detection? private bool _mouthfound = false; private bool _facefound = false; // debug logic private gameobject _previewtoggle; private gameobject _previewscreen; private image _previewimage; private facedetpreview _previewscript; private bool _debugpreviewon = false; private sprite _webcamsprite; private string _camerainf = ""; private gameobject _cameralist; private debugcameras _cameralistscript; private string _selectedcamera = ""; // create array of opencv rectangles array of faces. andy opencvforunity.rect[] rects; // create array of opencv rectangles array of mouths. opencvforunity.rect[] rectsm; // detection logic public int logicsolution = 1; // solution using? 0 = brightness, 1 = find/lose mouth. private bool _capturingface = false; public bool mouthisopenboolean { { return _mouthisopenboolean; } set { // make sure we've changed. if (_mouthisopenboolean==value) return; _mouthisopenboolean = value; // add chew if we've changed false if (value == false) { gameobject chewtext = gameobject.find ("chews"); chewing chewing = chewtext.getcomponent<chewing>(); chewing.addchew(); } } } // use initialization void start () { // debug preview toggle , image _previewtoggle = gameobject.find ("facepreviewtoggle"); _previewscript = _previewtoggle.getcomponent<facedetpreview> (); _previewscreen = gameobject.find ("facepreview"); _previewimage = _previewscreen.getcomponent<image> (); _cameralist = gameobject.find ("cameras"); if(_cameralist!=null) _cameralistscript = _cameralist.getcomponent<debugcameras>(); // setup camera , texture startcoroutine (init ()); } private ienumerator init () { // have initialised? restart if have. if (_webcamtexture != null) { _webcamtexture.stop (); _initdone = false; _rgbamat.dispose (); _graymat.dispose (); } // checks how many , cameras available on device _camerainf = "cameras=" + webcamtexture.devices.length.tostring (); (int cameraindex = 0; cameraindex < webcamtexture.devices.length; cameraindex++) { // debug information addtodebug( "camera " + cameraindex.tostring() + " - " + webcamtexture.devices [cameraindex].name + " - front:" + webcamtexture.devices [cameraindex].isfrontfacing); // want using correct camera, make sure it's front facing if (webcamtexture.devices [cameraindex].isfrontfacing == isfrontfacing) { // tell camera we're using debug.log (cameraindex + " name " + webcamtexture.devices [cameraindex].name + " isfrontfacing " + webcamtexture.devices [cameraindex].isfrontfacing); debug.log("here"); // set camera texture _webcamtexture = new webcamtexture (webcamtexture.devices [cameraindex].name, webcamtexturewidth, webcamtextureheight); break; } else { // want first 1 if (_firstcam=="") { _firstcam=webcamtexture.devices [cameraindex].name; } } } // did find camera? if (_webcamtexture == null) { // if we're in unity ok... #if unity_editor // create new camera _webcamtexture = new webcamtexture (_firstcam, webcamtexturewidth, webcamtextureheight); #endif // ****** need stop here!!!!!! *************************************************************************** // todo: device has no forward facing camera // can't chew without front facing camera!!! } // debug... addtodebug ("selected camera: " + _webcamtexture.devicename); debug.log(_webcamtexture.devicename); // how many fps want?? need control performance. although, request ignored! :/ _webcamtexture.requestedfps = webcamtexturefps; // start camera _webcamtexture.play (); while (true) { //if want use webcamtexture.width , webcamtexture.height on ios, have wait until webcamtexture.didupdatethisframe == 1, otherwise these 2 values equal 16. (http://forum.unity3d.com/threads/webcamtexture-and-error-0x0502.123922/) if (_webcamtexture.width > 16 && _webcamtexture.height > 16) { // create new colour , matrices _colors = new color32[_webcamtexture.width * _webcamtexture.height]; _rgbamat = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc4); _graymat = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc1); _mouthmatrgba = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc4); _mouthmatgray = new mat (_webcamtexture.height, _webcamtexture.width, cvtype.cv_8uc1); _texture = new texture2d (_webcamtexture.width, _webcamtexture.height, textureformat.rgba32, false); _textmouth = new texture2d (_webcamtexture.width, _webcamtexture.height, textureformat.rgba32, false); // set rotation // gameobject.transform.eulerangles = new vector3 (0, 0, 0); // android , iphone rotation needs change correct z axis /* #if (unity_android || unity_iphone) && !unity_editor gameobject.transform.eulerangles = new vector3 (0, 0, -90); #endif */ // i'm not sure why removed in sample, i'll leave out until figure out it's ueseful for, // looks developer trying dynamically correct rotated webcams. // gameobject.transform.rotation = gameobject.transform.rotation * quaternion.angleaxis (_webcamtexture.videorotationangle, vector3.back); // looks dynamically fixing scale???? why dev removing this??? may re-introduce it. /* bool videoverticallymirrored = _webcamtexture.videoverticallymirrored; float scalex = 1; float scaley = videoverticallymirrored ? -1.0f : 1.0f; if (_webcamtexture.videorotationangle == 270) scaley = -1.0f; gameobject.transform.localscale = new vector3 (scalex * gameobject.transform.localscale.x, scaley * gameobject.transform.localscale.y, 1); */ // create new classifiers - todo have work path make work android , iphone... _cascadeface = new cascadeclassifier (utils.getfilepath ("haarcascade_frontalface_alt.xml")); _cascademouth = new cascadeclassifier (utils.getfilepath ("haarcascade_mouth.xml")); // create matrix of faces (we'll load @ point???....) _faces = new matofrect (); _mouths = new matofrect (); // set camera ortho size, don't this work frog camera. camera.main.orthographicsize = _webcamtexture.width / 2; // we're initilised _initdone = true; break; } else { // come , try again, we're not ready yet. yield return 0; } } } // update called once per frame void update () { // make sure we've initialised. if ((_initdone) && (!_capturingface)) { startcoroutine ("captureface"); } } private ienumerator captureface() { _capturingface = true; // make sure texture has been correctly formated, if not we'll have come later. if (_webcamtexture.width > 16 && _webcamtexture.height > 16) { // pass web cam texture opencv matrix utils.webcamtexturetomat (_webcamtexture, _rgbamat, _colors); // iphones buggering mirroring again... #if unity_iphone && !unity_editor // flip if neccessary if (_webcamtexture.videoverticallymirrored){ if(isfrontfacing){ core.flip (_rgbamat, _rgbamat, 1); }else{ core.flip (_rgbamat, _rgbamat, 0); } }else{ if(isfrontfacing){ core.flip (_rgbamat, _rgbamat, -1); } } #endif // convert rgb web texture matrix gray imgproc.cvtcolor (_rgbamat, _graymat, imgproc.color_rgba2gray); // adjust contrast - can impact performance, try without faster performance imgproc.equalizehist (_graymat, _graymat); // set cascade detect different sized targets if (_cascadeface != null) _cascadeface.detectmultiscale (_graymat, _faces, 1.1, 2, 2, // todo: objdetect.cv_haar_scale_image new size (_webcamtexture.width * 0.15, _webcamtexture.width * 0.15), new size ()); // create array of opencv rectangles array of faces. rects = _faces.toarray (); // find mouth in each face each face. _facefound = false; (int = 0; < rects.length; i++) { // found face _facefound = true; // debugging, show face is. core.rectangle ( _rgbamat, new point (rects [i].x ,rects [i].y), new point (rects [i].x + rects [i].width, rects [i].y + rects [i].height), new scalar (0, 255, 0, 255), 2); // create rectangle around region of face we're interested in // keep inside face box, otherwise errors when we're @ edge of screen /* opencvforunity.rect _mouthroi = new opencvforunity.rect( (int)rects [i].x, (int)rects [i].y + (rects [i].height / 2), (int)rects [i].width, (int)(rects [i].height- (rects [i].height / 2))); */ opencvforunity.rect _mouthroi = new opencvforunity.rect( // improved sensitivity!!!! (int)rects [i].x, (int)rects [i].y + ((rects [i].height / 3) * 2), (int)rects [i].width, (int)(rects [i].height - ((rects [i].height / 3) * 2))); // create new matrix using roi _mouthmatgray = _graymat.submat(_mouthroi); _mouthmatrgba = _rgbamat.submat(_mouthroi); // detect mouth (we're going use first mouth captured // set cascade detect different sized targets if (_cascademouth != null) _cascademouth.detectmultiscale (_mouthmatgray, _mouths, 1.1, 2, 2, // todo: objdetect.cv_haar_scale_image new size (_webcamtexture.width * 0.04, _webcamtexture.width * 0.04), new size ()); // create array of opencv rectangles array of mouths. rectsm = _mouths.toarray (); // put rectangle around first mouth on each face _mouthfound = false; (int j = 0; j < rectsm.length; j++) { // set mouth box , make sure x , y correct (remember, these co-ords inside roi) _rectmouth = new unityengine.rect( rectsm[j].x + _mouthroi.x, _webcamtexture.height - _mouthroi.y - rectsm[j].y - rectsm[j].height, rectsm[j].width, rectsm[j].height ); // we've found mouth! _mouthfound = true; core.rectangle ( _rgbamat, new point (rectsm [j].x + _mouthroi.x,rectsm [j].y + _mouthroi.y), new point (rectsm [j].x + rectsm [j].width + _mouthroi.x, rectsm [j].y + rectsm [j].height + _mouthroi.y), new scalar (0, 0, 255, 255), 2); break; } } // add output asssigned texture! utils.mattotexture2d (_rgbamat, _texture, _colors); // debugging? _debugpreviewon = _previewscript.istoggled; if (_debugpreviewon) { _webcamsprite = sprite.create( _texture, new unityengine.rect( 0, 0, _texture.width, _texture.height), new vector2(0,0)); _previewimage.sprite = _webcamsprite; } //debug.log("face:" + _facefound.tostring() + " mouth:" + _mouthfound.tostring()); // version using?? fact have mouth satisfy "find/lose mouth" method if (logicsolution==1) { // we're setting oposite of mouth found (ie, if we've found mouth it's closed mouthisopenboolean = !_mouthfound; } // continue solution 0 , have mouth. if ((logicsolution==0) && (_mouthfound==true)) { debug.log("we shouldn't see this"); // how high , wide mouth _mouthwidth = (int)_rectmouth.width; _mouthheight = (int)_rectmouth.height; // create mouth texture color[] pixels = _texture.getpixels( (int)_rectmouth.x, (int)_rectmouth.y, (int)_rectmouth.width, (int)_rectmouth.height); _textmouth.resize(_mouthwidth, _mouthheight); _textmouth.apply(); _textmouth.setpixels(pixels); _textmouth.apply(); // work out detection pixels int mouthcentre = _mouthwidth / 2; _totalbright = 0; int mouthgap = _mouthheight / mouthdetectpixelcount; unityengine.rect rectshapepreview = new unityengine.rect((float)mouthcentre - 1, (float)mouthgap,3, (float)mouthgap*(float)mouthdetectpixelcount); // work out gap between pixels (int = 1; <= mouthdetectpixelcount; i++) { // work out total brightness int pixely = mouthgap * i; _totalbright += brightness(_textmouth.getpixel(mouthcentre, pixely)); } // have opened or closed. // // rules (for boolean): // 1. initial change 2 tenths in eather direction (double mouthchangediff) change open or closed. // smaller = open, bigger = closed. // 2. record current number _lastbright (double) , start closed (bool - false) in mouthisopenboolean // 3. if number gets bigger keep updating _lastbright variable make note of "close" brightness. // 4. if number gets smaller measure it, if it's more 2/10s (or ever variable we're open: // change bool true , record number. // if it's less leave it, if capture mouth when it's mid open or mid close. // 5. if we've moved closed stops to 4 in oposite direction // // rules (for integer) // 1. use boolean logic change diff maximimum 100 (open - oposite boolean :s) , minimum 0 (closed); // 2. use increment tie setting nearest increment. // record current number _lastbright if first capture if (_lastbright == 0) { _lastbright = _totalbright; } else { //get difference double diff = new double(); diff = _lastbright - _totalbright; // need positive diff double posdiff = diff; if (posdiff < 0) {posdiff = -posdiff;} // have gone more open or more closed? if (((_lastbright > _totalbright) && (mouthisopenboolean)) || ((_lastbright < _totalbright) && (!mouthisopenboolean))) { // we're going further in same direction, log last brighness _lastbright = _totalbright; } else { // we've changed direction. if more diff level set??? if (posdiff > mouthchangediff) { // we've changed direction! if (mouthisopenboolean) { mouthisopenboolean = false; } else { mouthisopenboolean = true; } // set new last brightness _lastbright = _totalbright; } } } } } yield return new waitforseconds (0.1f); // massive performance boost. _capturingface = false; yield return null; } void ondisable () { // stop webcam. _webcamtexture.stop (); } void ongui () { float screenscale = screen.width / 240.0f; matrix4x4 scaledmatrix = matrix4x4.scale (new vector3 (screenscale, screenscale, screenscale)); gui.matrix = scaledmatrix; // how big our gui box? // unityengine.rect devrect = new unityengine.rect (1, 1, (_texture.width/10)+2, (_texture.height/10)+90); } // return brighness of colour of pixel private double brightness(color c) { return mathf.sqrt ( c.r * c.r * .241f + c.g * c.g * .691f + c.b * c.b * .068f); } public void addtodebug(string newline) { // need add carraige return if (_camerainf != "") _camerainf += "\n"; // add line , update _camerainf += newline; if (_cameralistscript!=null) _cameralistscript.cameralist = _camerainf; } } specifically in update() ienumerator captureface() taking 150ms plus run according profiler.
because use cpu-heavy functions,
_textmouth.setpixels(pixels); _textmouth.apply(); and it's not platform-dependent issue.
Comments
Post a Comment