iink SDK on Device

Answered

Off-screen usage: no text recognized

Hello,

I'm using the iOS SDK and I have a really basic implementation of an off-screen handwriting recognizer here (uses all of the helpers from the reference implementation which are not shown)  

class HandwritingRecognizer {
    var editor: IINKEditor

    init(editor: IINKEditor) {
        self.editor = editor
    }

    func pointerDown(xStart: Double, yStart: Double) {
        var error: NSError?
        self.editor.pointerDown(
            CGPoint(x: xStart, y: yStart),
            at: -1,
            force: 1.0,
            type: .pen,
            pointerId: -1,
            error: &error
        )
        if error != nil {
            os_log(.error, "Could not set pointer down: %@", error!.localizedDescription)
        }
    }

    func pointerMove(xNew: Double, yNew: Double) {
        try! self.editor.pointerMove(
            CGPoint(x: xNew, y: yNew),
            at: -1,
            force: 1.0,
            type: .pen,
            pointerId: -1
        )
    }

    func pointerUp(xEnd: Double, yEnd: Double) {
        try! self.editor.pointerUp(
            CGPoint(x: xEnd, y: yEnd),
            at: -1,
            force: 1.0,
            type: .pen,
            pointerId: -1)
    }

    func generateText() -> String {
        self.editor.waitForIdle()
        try! self.editor.convert(editor.rootBlock, targetState: .digitalEdit)
        let result = try! self.editor.export_(self.editor.rootBlock, mimeType: .text)
        self.editor.clear()
        return result
    }
}

func initIInkEngine() -> IINKEngine? {
    // Check that the MyScript certificate is present
    os_log(.error, "test %@", "Hello test")
    if myCertificate.length == 0 {
        os_log(.error, "Could not locate MyScript certificate")
        return nil
    }

    // Create the iink runtime environment
    let data = Data(bytes: myCertificate.bytes, count: myCertificate.length)
    guard let engine = IINKEngine(certificate: data) else {
        os_log(.error, "Invalid MyScript certificate")
        return nil
    }

    // Configure the iink runtime environment
    let configurationPath = Bundle.main.bundlePath.appending("/recognition-assets/conf")
    do {
        try engine.configuration.setStringArray(
            [configurationPath], forKey: "configuration-manager.search-path"
        ) // Tells the engine where to load the recognition assets from.
    } catch {
        os_log(.error, "MyScript error, please check your resources assets: %@", error.localizedDescription)
        return nil
    }

    // Set the temporary directory
    do {
        try engine.configuration.setString(NSTemporaryDirectory(), forKey: "content-package.temp-folder")
    } catch {
        os_log(.error, "Failed to set temporary folder: %@", error.localizedDescription)
        return nil
    }

    do {
        // Recommended for off-screen usage
        try engine.configuration.setBoolean(false, forKey: "text.guides.enable")
    } catch {
        os_log(.error, "Failed disable myscript guides: %@", error.localizedDescription)
        return nil
    }

    return engine
}

func initEditor(engine: IINKEngine) -> IINKEditor? {
    let renderer = try! engine.createRenderer(withDpiX: 20, dpiY: 20, target: nil, error: ())
    guard let editor = engine.createEditor(renderer) else {
        return nil
    }
    let fontMetricsProvider = FontMetricsProvider()
    editor.setFontMetricsProvider(fontMetricsProvider)
    var error: NSError?
    editor.setViewSize(CGSize(width: 200, height: 200), error: &error)
    if error != nil {
        os_log(.error, "Failed to set view size: %@", error!.localizedDescription)
        return nil
    }

    let package = try! engine.createPackage("text.iink")
    let part = try! package.createPart("Text")
    editor.setPart(part, error: &error)
    if error != nil {
        os_log(.error, "Could not set MyScript Part: %@", error!.localizedDescription)
        return nil
    }
    return editor
}

  And I have a simple test case here:

 

class HandwritingRecognitionTests: XCTestCase {
    var editor: IINKEditor!
    var engine: IINKEngine!
    var recognizer: HandwritingRecognizer!

    override func setUpWithError() throws {
        self.engine = initIInkEngine()
        self.editor = initEditor(engine: self.engine)!
        self.recognizer = HandwritingRecognizer(editor: self.editor)
    }

    func test_generateText_shouldReturnExpectedText() throws {
        for stroke in testPlusStrokes {
            stroke.write(recognizer: recognizer)
        }
        XCTAssertEqual(recognizer.generateText(), "+")
     }

}

struct Stroke {
    var start: (Double, Double)
    var points: [(Double, Double)]
    var end: (Double, Double)

    func write(recognizer: HandwritingRecognizer) {
        recognizer.pointerDown(xStart: start.0, yStart: start.1
        )
        for (xNew, yNew) in points {
            recognizer.pointerMove(
                xNew: xNew,
                yNew: yNew
            )
        }
        recognizer.pointerUp(xEnd: end.0, yEnd: end.1)

    }
}

let upStroke = Stroke(
    start: (184.0, 123.0),
    points: [
        (184.0, 124.0),
        (184.0, 125.0),
        (184.0, 128.0),
        (184.0, 133.0),
        (184.0, 152.0),
        (184.0, 158.0),
        (184.0, 163.0),
        (184.0, 167.0),
        (184.0, 174.0),
        (184.0, 183.0),
        (184.0, 184.0)
    ],
    end: (183.0, 185.0)
)

let rightStroke = Stroke(
    start: (149.5, 150.0),
    points: [
        (150.0, 150.0),
        (151.0, 150.0),
        (152.0, 150.0),
        (158.0, 150.0),
        (166.0, 150.0),
        (184.0, 150.0),
        (190.0, 150.0),
        (196.0, 150.0),
        (200.0, 150.0),
        (207.0, 150.0),
        (208.0, 150.0),
        (209.0, 150.0)
    ],
    end: (209.5, 1150.0)
)

let testPlusStrokes = [upStroke, rightStroke]

The test fails as the generated text is always an empty string. There are no errors otherwise. Am I missing something? Does this implementation look reasonable?

Thanks!


Best Answer

Figured it out, my recognition-assets directory needed to be a folder reference when it was a group.

Thanks for the help.

As a matter of feedback, it would have been nice to get an informative error about this somewhere rather than just returning an empty string when I try to run recognition.


Dear Claire,


Thank you for contacting us.


Without your full project it is quite difficult to fully understand what is going on.


Nevertheless, I would suggest you to try following modifications in the generateText function to understand where the issue comes from:

After the waitforIdle, add another export before calling the convert method, to log the recognition result before the convert operation. Then add a waitforIdle between this export and the convert call.


For the conversion, retrieve the conversion state as in the code sample, instead of using  the

targetState: .digitalEdit, for instance:


let supportedTargetStates = editorViewController.editor.getSupportedTargetConversionState(nil)

try editorViewController.editor.convert(nil, targetState: supportedTargetStates[0].value)


In a test phase, I would suggest you to comment the self.editor.clear() call too, as it empties the part content.


If these modifications do not help you fixing your problem, can you tell us which version you are using?

Which recognition language are you using?


Best regards,


Gwenaëlle

Thanks for the help. Unfortunately, this didn't solve my problem.

I set up an example project that reproduces my issue by forking off the GetStartedSwift example, but to my surprise the exact same code works in that project but not in mine.

Any tips on debugging this?

I didn't want to commit the entire reference implementation to my project so instead I have this pod installed

 pod 'MyScriptInteractiveInk-Runtime'

And I copied in the following helper files from the reference implementation:


NSAttributedString+Helper.h
UIFont+Traits.h 
UIfont+Helper.h
NSAttributedString+Helper.mm 
UIFont+Traits.m 
UIfont+Helper.mm
FontMetricsProvider.h
FontMetricsProvider.mm

and of course the certificate. Is there anything else I may have forgotten to include that could be causing this?



Answer

Figured it out, my recognition-assets directory needed to be a folder reference when it was a group.

Thanks for the help.

As a matter of feedback, it would have been nice to get an informative error about this somewhere rather than just returning an empty string when I try to run recognition.

Dear Claire,


Thank you for your feedback and suggestion.


Best regards,


Gwenaëlle