BoggleCV/minimal-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2, math, os, json, traceback, io, time\n",
    "import numpy as np\n",
    "# import matplotlib.pyplot as plt\n",
    "import scipy.signal\n",
    "\n",
    "print(cv2.__version__)\n",
    "\n",
    "RED = (0, 0, 255)\n",
    "BLUE = (255, 0, 0)\n",
    "GREEN = (0, 255, 0)\n",
    "YELLOW = (0, 255, 255)\n",
    "CONTOUR_THICKNESS = 2\n",
    "MAX_DISP_DIM = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#a generic error\n",
    "class BoggleError(Exception):\n",
    "    def __init__(self, arg):\n",
    "        self.strerror = arg\n",
    "        self.args = {arg}\n",
    "\n",
    "def four_point_transform(image, pts, size):\n",
    "    w = size\n",
    "    h = size\n",
    "    ntl = [0, 0]\n",
    "    ntr = [w, 0]\n",
    "    nbr = [w, h]\n",
    "    nbl = [0, h]\n",
    "    location = np.float32(pts)\n",
    "    x0 = pts[0][0]\n",
    "    x2 = pts[2][0]\n",
    "    #make sure the board ends up rotated the correct way\n",
    "    if x0 < x2:\n",
    "        newLocation = np.float32([ntl, nbl, nbr, ntr])\n",
    "    else:\n",
    "        newLocation = np.float32([ntr, ntl, nbl, nbr])\n",
    "    M = cv2.getPerspectiveTransform(location, newLocation)\n",
    "    warpedimage = cv2.warpPerspective(image, M, (w, h))\n",
    "    return warpedimage\n",
    "\n",
    "def resizeWithAspectRatio(image, maxDispDim, inter=cv2.INTER_AREA):\n",
    "    w = image.shape[0] #TODO width and height are swapped here\n",
    "    h = image.shape[1]\n",
    "\n",
    "    ar = w / h\n",
    "    #print(ar)\n",
    "    if w > h:\n",
    "        nw = maxDispDim\n",
    "        nh = int(maxDispDim / ar)\n",
    "    elif h > w:\n",
    "        nh = maxDispDim\n",
    "        nw = int(maxDispDim / ar)\n",
    "    else:\n",
    "        nh = maxDispDim\n",
    "        nw = maxDispDim\n",
    "\n",
    "    dim = None\n",
    "    (h, w) = image.shape[:2]\n",
    "\n",
    "    r = nw / float(w)\n",
    "    dim = (nw, int(h * r))\n",
    "\n",
    "    return cv2.resize(image, dim, interpolation=inter)\n",
    "\n",
    "def imshow_fit(name, img, maxDispDim=MAX_DISP_DIM):\n",
    "    if maxDispDim is not None:\n",
    "        img = resizeWithAspectRatio(img, maxDispDim)\n",
    "    cv2.imshow(name, img)\n",
    "\n",
    "def findBestCtr(contours):\n",
    "    bestArea = 0\n",
    "    bestCtr = None\n",
    "    for i, ctr in enumerate(contours):\n",
    "        area = cv2.contourArea(ctr)\n",
    "        # perimeter = cv2.arcLength(ctr, True)\n",
    "        if area > bestArea:\n",
    "            bestArea = area\n",
    "            bestCtr = ctr\n",
    "    return bestCtr\n",
    "\n",
    "\n",
    "def angleEveryFew(ctr, step):\n",
    "    angles = []\n",
    "    # dists = []\n",
    "    prev = ctr[-1]\n",
    "    for i in range(0, len(ctr), step):\n",
    "        curr = ctr[i]\n",
    "        px, py = prev[0]\n",
    "        cx, cy = curr[0]\n",
    "        # angle = (px-cx)/(py-cy)\n",
    "        angle = math.atan2(cy - py, cx - px)\n",
    "        angles.append(angle)\n",
    "        # dist = math.hypot(cx-px, cy-py)\n",
    "        # dists.append(dist)\n",
    "        prev = curr\n",
    "    return angles\n",
    "\n",
    "\n",
    "def angleAvg(angles):\n",
    "    x = 0\n",
    "    y = 0\n",
    "    for angle in angles:\n",
    "        x += math.cos(angle)\n",
    "        y += math.sin(angle)\n",
    "    return math.atan2(y, x)\n",
    "\n",
    "\n",
    "def angleDiffAbs(angle1, angle2):\n",
    "    return abs(((angle1 - angle2 + math.pi) % (2 * math.pi)) - math.pi)\n",
    "\n",
    "\n",
    "def runningAvg(angles, history):\n",
    "    result = []\n",
    "    for i in range(len(angles)):\n",
    "        # avg = 0\n",
    "        vals2 = []\n",
    "        for j in range(history):\n",
    "            val = angles[int(i + j - history / 2) % len(angles)]\n",
    "            vals2.append(val)\n",
    "            # avg += val\n",
    "        # avg /= history\n",
    "        avg = angleAvg(vals2)\n",
    "        result.append(avg)\n",
    "    return result\n",
    "\n",
    "\n",
    "def diffAbs(vals):\n",
    "    diffs = []\n",
    "    prev = vals[-1]\n",
    "    for i in range(0, len(vals), 1):\n",
    "        curr = vals[i]\n",
    "        diff = angleDiffAbs(prev, curr) * 10\n",
    "        diffs.append(diff)\n",
    "        prev = curr\n",
    "    return diffs\n",
    "\n",
    "\n",
    "def debounce(bools, history):\n",
    "    result = []\n",
    "    for i in range(len(bools)):\n",
    "        total = 0\n",
    "        for j in range(history):\n",
    "            val = bools[int(i + j - history / 2) % len(bools)]\n",
    "            total += val\n",
    "        result.append(int(total >= history / 2))\n",
    "    return result\n",
    "\n",
    "\n",
    "def findGaps(diffs):\n",
    "    wasGap = True\n",
    "    seamI = 0\n",
    "    for i, diff in enumerate(diffs):\n",
    "        isGap = diff\n",
    "        if not isGap and not wasGap:\n",
    "            seamI = i\n",
    "            break\n",
    "        wasGap = isGap\n",
    "\n",
    "    xs = range(len(diffs))\n",
    "    xs = [x for x in xs]\n",
    "    xs_a = xs[seamI:]\n",
    "    xs_a.extend(xs[:seamI])\n",
    "    diffs_a = diffs[seamI:]\n",
    "    diffs_a.extend(diffs[:seamI])\n",
    "\n",
    "    xs2 = []\n",
    "    diffs2 = []\n",
    "    gapsStart = []\n",
    "    gapsStartY = []\n",
    "    gapsEnd = []\n",
    "    gapsEndY = []\n",
    "    wasGap = None\n",
    "    gapwidth = 0\n",
    "    for x, diff in zip(xs_a, diffs_a):\n",
    "        isGap = diff\n",
    "        if wasGap is None:\n",
    "            wasGap = isGap\n",
    "        if isGap:\n",
    "            xs2.append(x)\n",
    "            diffs2.append(diff)\n",
    "            gapwidth += 1\n",
    "        if isGap and not wasGap:\n",
    "            gapsStart.append(x)\n",
    "            gapsStartY.append(.5)\n",
    "        if not isGap and wasGap:\n",
    "            gapsEnd.append(x)\n",
    "            gapsEndY.append(gapwidth)\n",
    "            gapwidth = 0\n",
    "        wasGap = isGap\n",
    "    return gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2\n",
    "    # gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]\n",
    "    # return gaps, diffs, xs2\n",
    "\n",
    "\n",
    "def top4gaps(gaps):\n",
    "    def length_sort(gap):\n",
    "        return -gap[2]\n",
    "\n",
    "    gaps2 = sorted(gaps, key=length_sort)\n",
    "    gaps2 = gaps2[:4]\n",
    "    return [i for i in gaps if i in gaps2]\n",
    "\n",
    "\n",
    "def invertGaps(gaps):\n",
    "    segments = []\n",
    "    prev = gaps[-1]\n",
    "    for curr in gaps:\n",
    "        segments.append((prev[1], curr[0]))\n",
    "        prev = curr\n",
    "    return segments\n",
    "\n",
    "\n",
    "def findSidePoints(segments, ctr, step):\n",
    "    sidePoints = []\n",
    "    for seg in segments:\n",
    "        if seg[1] > seg[0]:\n",
    "            sidePoints.append(ctr[seg[0] * step:seg[1] * step])\n",
    "        else:\n",
    "            sidePoints.append(ctr[seg[0] * step:, :seg[1] * step])\n",
    "    return sidePoints\n",
    "\n",
    "\n",
    "def getEndVals(arr, fraction):\n",
    "    if fraction >= 0.5: return arr\n",
    "    l = len(arr)\n",
    "    keep = int(fraction * l)\n",
    "    keep = max(keep, 1)\n",
    "    return np.concatenate((arr[:keep], arr[l - keep:]))\n",
    "\n",
    "\n",
    "def fitSidePointsToLines(sidePoints):\n",
    "    lines = []\n",
    "    for sp in sidePoints:\n",
    "        xs = np.zeros(len(sp), int)\n",
    "        ys = np.zeros(len(sp), int)\n",
    "        for i, pt in enumerate(sp):\n",
    "            x, y = pt[0] #TODO if pt is empty\n",
    "            xs[i] = x\n",
    "            ys[i] = y\n",
    "        lines.append(np.polyfit(xs, ys, 1))\n",
    "    return lines\n",
    "\n",
    "\n",
    "def findCorners(lines):\n",
    "    points = []\n",
    "\n",
    "    prev = lines[-1]\n",
    "    for curr in lines:\n",
    "        a1, b1 = prev\n",
    "        a2, b2 = curr\n",
    "\n",
    "        x = (b2 - b1) / (a1 - a2)\n",
    "        y = np.polyval(curr, x)\n",
    "        #if math.isnan(x): x = 0 #TODO\n",
    "        #if math.isnan(y): y = 0\n",
    "        points.append((int(x), int(y)))\n",
    "        prev = curr\n",
    "    return points\n",
    "\n",
    "\n",
    "def drawLinesAndPoints(image, lines, points):\n",
    "    width = image.shape[1]\n",
    "\n",
    "    for l in lines:\n",
    "        y1 = int(np.polyval(l, 0))\n",
    "        y2 = int(np.polyval(l, width - 1))\n",
    "        cv2.line(image, (0, y1), (width - 1, y2), RED, CONTOUR_THICKNESS)\n",
    "\n",
    "    for point in points:\n",
    "        cv2.circle(image, point, 4, BLUE, 3)\n",
    "\n",
    "#https://scipy-cookbook.readthedocs.io/items/SignalSmooth.html\n",
    "#window: np.ones (flat), np.hanning, np.hamming, np.bartlett, np.blackman\n",
    "def smooth(x, window_len=11, window=np.hanning):\n",
    "    if x.ndim != 1:\n",
    "        raise ValueError(\"smooth only accepts 1 dimension arrays.\")\n",
    "\n",
    "    if x.size < window_len:\n",
    "        raise ValueError(\"Input vector needs to be bigger than window size.\")\n",
    "\n",
    "    if window_len<3:\n",
    "        return x\n",
    "\n",
    "    s=np.r_[x[window_len-1:0:-1], x, x[-2:-window_len-1:-1]]\n",
    "    w = window(window_len)\n",
    "    y = np.convolve(w / w.sum(), s, mode='valid')\n",
    "    return y\n",
    "\n",
    "\n",
    "def findRowsOrCols(img, doCols, smoothFactor, ax):\n",
    "    smoothFactor = int(smoothFactor * img.shape[0])\n",
    "    #print(\"smoothFactor\", smoothFactor)\n",
    "    \n",
    "    if doCols:\n",
    "        title = \"colSum\"\n",
    "        imgSum = cv2.reduce(img, 0, cv2.REDUCE_AVG, dtype=cv2.CV_32S)\n",
    "        imgSum = imgSum[0]\n",
    "    else:\n",
    "        #row sum\n",
    "        title = \"rowSum\"\n",
    "        imgSum = cv2.reduce(img, 1, cv2.REDUCE_AVG, dtype=cv2.CV_32S)\n",
    "        imgSum = imgSum.reshape(len(imgSum))\n",
    "    \n",
    "    imgSumSmooth = smooth(imgSum, smoothFactor*2)\n",
    "\n",
    "    #https://qingkaikong.blogspot.com/2018/07/find-peaks-in-data.html\n",
    "    #peaks_positive, _ = scipy.signal.find_peaks(imgSumSmooth, height=200, threshold = None, distance=60)\n",
    "    dips, props = scipy.signal.find_peaks(-imgSumSmooth, height=(None,None), distance=30, prominence=(None,None))\n",
    "    \n",
    "    #threshold=(None,None), \n",
    "    #, plateau_size=(None,None)\n",
    "    \n",
    "    #print(props)\n",
    "\n",
    "    prs = props[\"prominences\"]\n",
    "    if len(prs) < 6:\n",
    "        top_6_dips = dips\n",
    "        #print (\"!!!! less than 6 dips\")\n",
    "        #raise BoggleError(\"less than 6 dips\")\n",
    "    else:\n",
    "        prsIdx = sorted(range(len(prs)), key=lambda i: prs[i], reverse=True)\n",
    "        #print(prsIdx)\n",
    "        prsIdx = prsIdx[:6]\n",
    "        #print(prsIdx)\n",
    "        top_6_dips = [p for i,p in enumerate(dips) if i in prsIdx]\n",
    "\n",
    "    #fig = plt.figure()\n",
    "    #ax1 = fig.add_subplot(111)\n",
    "    \n",
    "    if ax is not None:\n",
    "        q = [i for i in range(len(imgSumSmooth))]\n",
    "        \n",
    "        ax.plot(q, imgSumSmooth, 'b-', linewidth=2, label=\"smooth\")\n",
    "        ax.plot(np.linspace(smoothFactor,len(imgSumSmooth)-smoothFactor, len(imgSum)), imgSum, 'r-', linewidth=1, label=title)\n",
    "\n",
    "        #ax.plot(\n",
    "            #[q[p] for p in peaks_positive],\n",
    "            #[imgSumSmooth[p] for p in peaks_positive],\n",
    "            #'ro', label = 'positive peaks')\n",
    "        \n",
    "        ax.plot(\n",
    "            [q[p] for p in dips],\n",
    "            [imgSumSmooth[p] for p in dips],\n",
    "            'go', label='dips')\n",
    "        \n",
    "        ax.plot(\n",
    "            [q[p] for p in top_6_dips],\n",
    "            [imgSumSmooth[p] for p in top_6_dips],\n",
    "            'c.', label='top 6 dips')\n",
    "        \n",
    "        ax.legend(loc='best')\n",
    "        \n",
    "    #return top_6_dips\n",
    "    #print(\"before clip\", top_6_dips)\n",
    "    top_6_dips_scaled = [np.clip(0, p-smoothFactor, len(imgSum)-1) for p in top_6_dips]\n",
    "    return top_6_dips_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findBoggleBoard(image, normalPlots=True, harshErrors=False, generate=(\"debugimage\", \"debugmask\", \"contourPlotImg\", \"warpedimage\", \"imgSumPlotImg\", \"diceRaw\", \"dice\")):\n",
    "    resultImages = {}\n",
    "\n",
    "    # maskThresholdMin = (108, 28, 12)\n",
    "    # maskThresholdMax = (125, 255, 241)\n",
    "    # maskThresholdMin = (108, 28, 6)\n",
    "    # maskThresholdMax = (130, 255, 241)\n",
    "    maskThresholdMin = (108, 28, 6)\n",
    "    maskThresholdMax = (144, 255, 241)\n",
    "    size = max(image.shape)\n",
    "    #print(\"size\", size)\n",
    "    blurAmount = int(.02 * size)\n",
    "    blurAmount = (blurAmount, blurAmount)\n",
    "    # blurThreshold = 80\n",
    "    blurThreshold = 40\n",
    "    contourApprox = cv2.CHAIN_APPROX_NONE\n",
    "    # contourApprox = cv2.CHAIN_APPROX_SIMPLE\n",
    "    # contourApprox = cv2.CHAIN_APPROX_TC89_L1\n",
    "    # contourApprox = cv2.CHAIN_APPROX_TC89_KCOS\n",
    "\n",
    "    hsvimg = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n",
    "    mask = cv2.inRange(hsvimg, maskThresholdMin, maskThresholdMax)\n",
    "\n",
    "    maskblur = cv2.blur(mask, blurAmount)\n",
    "    # maskblur = cv2.threshold(maskblur, 80, 255, cv2.THRESH_BINARY_INV)\n",
    "    maskblur = cv2.inRange(maskblur, (blurThreshold,), (255,))\n",
    "    \n",
    "    #API CHANGE: findContours no longer returns a modified image\n",
    "    #contourimg, contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)\n",
    "    contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)\n",
    "    # print(hierarchy) #TODO\n",
    "    \n",
    "    bestCtr = findBestCtr(contours)\n",
    "    # bestCtr = cv2.convexHull(bestCtr)\n",
    "\n",
    "    #draw the contours for debugging\n",
    "    if \"debugimage\" in generate:\n",
    "        debugimage = image.copy()\n",
    "        cv2.drawContours(debugimage, contours, -1, RED, CONTOUR_THICKNESS)\n",
    "        cv2.drawContours(debugimage, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)\n",
    "    if \"debugmask\" in generate:\n",
    "        debugmask = cv2.cvtColor(maskblur, cv2.COLOR_GRAY2BGR)\n",
    "        cv2.drawContours(debugmask, contours, -1, RED, CONTOUR_THICKNESS)\n",
    "        cv2.drawContours(debugmask, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)\n",
    "    \n",
    "\n",
    "    step = 10\n",
    "    avgWindow = 0.1\n",
    "    debounceFactor = .05\n",
    "    \n",
    "    angles = angleEveryFew(bestCtr, step)\n",
    "\n",
    "    xs = range(len(angles))\n",
    "\n",
    "    anglesAvg = runningAvg(angles, int(avgWindow * len(angles)))\n",
    "    diffs = diffAbs(anglesAvg)\n",
    "    \n",
    "    avgDiff = np.mean(diffs)\n",
    "    #print(avgDiff)\n",
    "    \n",
    "    binDiffs = [int(i > avgDiff) for i in diffs]\n",
    "    binDiffs = debounce(binDiffs, int(len(binDiffs) * debounceFactor))\n",
    "    gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2 = findGaps(binDiffs)\n",
    "    gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]\n",
    "    \n",
    "    #scale for viewing on the plot\n",
    "    gapsEndY2 = gapsEndY\n",
    "    if len(gapsEndY) > 0:\n",
    "        q = max(gapsEndY)\n",
    "        if q > 6: gapsEndY2 = [i * 6 / q for i in gapsEndY]\n",
    "\n",
    "    if \"contourPlotImg\" in generate:\n",
    "        contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, gapsEndY2, normalPlots)\n",
    "        if contourPlotImg is not None:\n",
    "            resultImages[\"contourPlotImg\"] = contourPlotImg\n",
    "    \n",
    "    if len(gaps) < 4:\n",
    "        print(\"!!!! less than 4 gaps\")\n",
    "        if harshErrors:\n",
    "            raise BoggleError(\"less than 4 gaps\")\n",
    "        if \"contourPlotImg\" in generate:\n",
    "            contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, None, normalPlots)\n",
    "            if contourPlotImg is not None:\n",
    "                resultImages[\"contourPlotImg\"] = contourPlotImg\n",
    "        \n",
    "        if \"debugmask\" in generate:\n",
    "            resultImages[\"debugmask\"] = debugmask\n",
    "        if \"debugimage\" in generate:\n",
    "            resultImages[\"debugimage\"] = debugimage\n",
    "        return resultImages, None\n",
    "    \n",
    "    endFraction = 0.01\n",
    "    \n",
    "    gaps = top4gaps(gaps)\n",
    "    segments = invertGaps(gaps)\n",
    "    sidePoints = findSidePoints(segments, bestCtr, step)\n",
    "    sidePoints = [getEndVals(sp, endFraction) for sp in sidePoints]\n",
    "    #print(\"sidepoints len\", len(sidePoints[0]))\n",
    "    \n",
    "    \n",
    "    lines = fitSidePointsToLines(sidePoints)\n",
    "    points = findCorners(lines)\n",
    "    if \"debugimage\" in generate:\n",
    "        cv2.drawContours(debugimage, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)\n",
    "        drawLinesAndPoints(debugimage, lines, points)\n",
    "        resultImages[\"debugimage\"] = debugimage\n",
    "    if \"debugmask\" in generate:\n",
    "        cv2.drawContours(debugmask, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)\n",
    "        drawLinesAndPoints(debugmask, lines, points)\n",
    "        resultImages[\"debugmask\"] = debugmask\n",
    "\n",
    "    npPoints = np.array(points)\n",
    "    size = 300\n",
    "    warpedimage = four_point_transform(image, npPoints, size)\n",
    "    warpgray = cv2.cvtColor(warpedimage, cv2.COLOR_BGR2GRAY)\n",
    "    \n",
    "    if \"warpedimage\" in generate:\n",
    "        resultImages[\"warpedimage\"] = warpedimage\n",
    "    \n",
    "    if \"warpgray\" in generate:\n",
    "        resultImages[\"warpgray\"] = warpgray\n",
    "    \n",
    "    smoothFactor = .05\n",
    "    \n",
    "    if \"imgSumPlotImg\" in generate:\n",
    "        fig, (ax0, ax1) = plt.subplots(2, figsize=(8,10))\n",
    "    else:\n",
    "        ax0 = ax1 = None\n",
    "    \n",
    "    rowSumLines = findRowsOrCols(warpgray, False, smoothFactor, ax0)\n",
    "    #print(\"rows\", rowSumLines)\n",
    "    colSumLines = findRowsOrCols(warpgray, True, smoothFactor, ax1)\n",
    "    #print(\"cols\", colSumLines)\n",
    "    \n",
    "    \n",
    "    if \"imgSumPlotImg\" in generate:\n",
    "        if normalPlots:\n",
    "            plt.show(block=False)\n",
    "        else:\n",
    "            resultImages[\"imgSumPlotImg\"] = plotToImg()\n",
    "\n",
    "\n",
    "    if len(rowSumLines) < 6 or len(colSumLines) < 6:\n",
    "        print(\"!!!! not enough grid lines\")\n",
    "        if harshErrors:\n",
    "            raise BoggleError(\"not enough gridlines\")\n",
    "        return resultImages, None\n",
    "    \n",
    "    #fix the outermost lines of the board\n",
    "    h1 = rowSumLines[2] - rowSumLines[1]\n",
    "    h2 = rowSumLines[3] - rowSumLines[2]\n",
    "    h3 = rowSumLines[4] - rowSumLines[3]\n",
    "    h = max(h1, h2, h3)\n",
    "    \n",
    "    newCSL0 = colSumLines[1] - h\n",
    "    if newCSL0 > colSumLines[0]:\n",
    "        colSumLines[0] = newCSL0\n",
    "    newCSL5 = colSumLines[4] + h\n",
    "    if newCSL5 < colSumLines[5]:\n",
    "        colSumLines[5] = newCSL5\n",
    "    \n",
    "    w1 = colSumLines[2] - colSumLines[1]\n",
    "    w2 = colSumLines[3] - colSumLines[2]\n",
    "    w3 = colSumLines[4] - colSumLines[3]\n",
    "    w = max(w1, w2, w3)\n",
    "    \n",
    "    newRSL0 = rowSumLines[1] - w\n",
    "    if newRSL0 > rowSumLines[0]:\n",
    "        rowSumLines[0] = newRSL0\n",
    "    newRSL5 = rowSumLines[4] + w\n",
    "    if newRSL5 < rowSumLines[5]:\n",
    "        rowSumLines[5] = newRSL5\n",
    "\n",
    "    #print(\"rows2\", rowSumLines)\n",
    "    #print(\"cols2\", colSumLines)\n",
    "\n",
    "    #just display\n",
    "    if \"diceRaw\" in generate:\n",
    "        plt.figure(figsize=(10,10))\n",
    "        i = 1\n",
    "        for y in range(5):\n",
    "            for x in range(5):\n",
    "                plt.subplot(5,5,i)\n",
    "                i += 1\n",
    "                plt.xticks([])\n",
    "                plt.yticks([])\n",
    "                plt.grid(False)\n",
    "                minX = colSumLines[x]\n",
    "                maxX = colSumLines[x+1]\n",
    "                minY = rowSumLines[y]\n",
    "                maxY = rowSumLines[y+1]\n",
    "                crop_img = warpgray[minY:maxY, minX:maxX]\n",
    "                plt.imshow(crop_img, cmap=plt.cm.gray)\n",
    "        if normalPlots:\n",
    "            plt.show(block=False)\n",
    "        else:\n",
    "            resultImages[\"diceRaw\"] = plotToImg()\n",
    "\n",
    "    \n",
    "    if \"dice\" in generate:\n",
    "        plt.figure(figsize=(10,10))\n",
    "        i = 1\n",
    "\n",
    "    letterResize = 30\n",
    "    #make square, resize, display, and save to an array\n",
    "    letterImgs = []\n",
    "    for y in range(5):\n",
    "        letterImgRow = []\n",
    "        for x in range(5):\n",
    "            minX = colSumLines[x]\n",
    "            maxX = colSumLines[x+1]\n",
    "            minY = rowSumLines[y]\n",
    "            maxY = rowSumLines[y+1]\n",
    "            w = maxX - minX\n",
    "            h = maxY - minY\n",
    "            #print(\"w,h 1:\", w, h)\n",
    "            d = abs(w - h)\n",
    "            if d > 0:\n",
    "                if int(d/2) == d/2:\n",
    "                    #even difference\n",
    "                    d1 = d2 = int(d/2)\n",
    "                else:\n",
    "                    #odd difference\n",
    "                    d1 = int((d-1)/2)\n",
    "                    d2 = int((d+1)/2)\n",
    "                if w > h:\n",
    "                    #wider than it is tall\n",
    "                    minX += d1\n",
    "                    maxX -= d2\n",
    "                else:\n",
    "                    #taller than it is wide\n",
    "                    minY += d1\n",
    "                    maxY -= d2\n",
    "            #print(\"w,h 2:\", maxX-minX, maxY-minY)\n",
    "            crop_img = warpgray[minY:maxY, minX:maxX]\n",
    "            letterImg = cv2.resize(crop_img, (letterResize,letterResize), interpolation=cv2.INTER_AREA)\n",
    "            if \"dice\" in generate:\n",
    "                plt.subplot(5,5,i)\n",
    "                i += 1\n",
    "                plt.xticks([])\n",
    "                plt.yticks([])\n",
    "                plt.grid(False)\n",
    "                plt.imshow(letterImg, cmap=plt.cm.gray)\n",
    "            letterImgRow.append(letterImg)\n",
    "        letterImgs.append(letterImgRow)\n",
    "\n",
    "    if \"dice\" in generate:\n",
    "        if normalPlots:\n",
    "            plt.show(block=False)\n",
    "        else:\n",
    "            resultImages[\"dice\"] = plotToImg()\n",
    "    \n",
    "    return resultImages, letterImgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://www.tensorflow.org/tutorials/keras/classification\n",
    "\n",
    "# from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "# TensorFlow and tf.keras\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import datasets, layers, models\n",
    "\n",
    "print(tf.__version__)\n",
    "\n",
    "# Helper libraries\n",
    "import numpy as np\n",
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "import json\n",
    "\n",
    "MODEL_FILE=\"/home/johanv/nextcloud/projects/boggle2.0/model.h5\"\n",
    "#MODEL_URL = \"https://drive.confuzer.cloud/index.php/s/HWSczZqSjbPqKDq/download\"\n",
    "#import os\n",
    "#os.system(\"curl \" + MODEL_URL + \" > \" + MODEL_FILE)\n",
    "\n",
    "IMG_DIM = 30\n",
    "\n",
    "class_names = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n",
    "\n",
    "L = len(class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(MODEL_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def processImage(filepath):\n",
    "    image = cv2.imread(filepath)\n",
    "    _, letters5x5grid = findBoggleBoard(image, normalPlots=False, harshErrors=True, generate=())\n",
    "    \n",
    "    lettersGuessed = \"\"\n",
    "    confidence = []\n",
    "    num_rows = 5\n",
    "    num_cols = 5\n",
    "    num_images = num_rows*num_cols\n",
    "    for row in range(num_rows):\n",
    "        for col in range(num_cols):\n",
    "            letterImg = letters5x5grid[row][col]\n",
    "            letterImg = letterImg / 255\n",
    "#             print(letterImg.shape)\n",
    "            letterImg = (np.expand_dims(letterImg,0))\n",
    "            letterImg = (np.expand_dims(letterImg,axis=3))\n",
    "#             print(letterImg.shape)\n",
    "            pred = model.predict(letterImg)[0]\n",
    "            letter = class_names[np.argmax(pred)]\n",
    "            lettersGuessed += letter\n",
    "            confidence.append(np.max(pred))\n",
    "    return lettersGuessed, confidence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# letters5x5gridLabelsStr = \"IZEIMFLTYTOSEINHETNORRISU\" #00162\n",
    "# letters5x5gridLabelsStr = \"DONLIEEIIESAPYWTAAKLTINRE\" #00164\n",
    "# letters5x5gridLabelsStr = \"RAOCODSEERGAPEWORXRHELWNT\" #00165\n",
    "# letters5x5gridLabelsStr = \"RONLICTSSDNMPPNUAEQIHINRM\" #00170\n",
    "letters5x5gridLabelsStr = \"VANUIRAUHESKEITTDPRCGOUCA\" #00171, 00160\n",
    "# letters5x5gridLabelsStr = \"IXESMFLEYTOSEENOETNRRRIWM\" #00172, 00161\n",
    "\n",
    "#process 1 image\n",
    "IMAGE_FILE = '/home/johanv/nextcloud/projects/boggle2.0/cascademan/categories/5x5/images/00160.jpg'\n",
    "lettersGuessed, confidence = processImage(IMAGE_FILE)\n",
    "\n",
    "# letters5x5gridLabels = [class_names.index(letter) for letter in letters5x5gridLabelsStr]\n",
    "right = \"\".join([str(int(a == b)) for a,b in zip(lettersGuessed, letters5x5gridLabelsStr)])\n",
    "\n",
    "confidence_right = []\n",
    "confidence_wrong = []\n",
    "\n",
    "for i, c in enumerate(confidence):\n",
    "    if right[i] == \"1\":\n",
    "        confidence_right.append(c)\n",
    "    else:\n",
    "        confidence_wrong.append(c)\n",
    "\n",
    "print(\"guess: \" + lettersGuessed)\n",
    "print(\"real:  \" + letters5x5gridLabelsStr)\n",
    "print(\"right: \" + right)\n",
    "# print(\"confidence:\", confidence)\n",
    "# print(\"confidence_right:\", confidence_right)\n",
    "# print(\"confidence_wrong:\", confidence_wrong)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}