Using OpenCV and Tensorflow to identify a Boggle board in an image and decode what letters are on the board.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

775 lines
28 KiB

{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import cv2, math, os, json, traceback, io, time\n",
"import numpy as np\n",
"# import matplotlib.pyplot as plt\n",
"import scipy.signal\n",
"\n",
"print(cv2.__version__)\n",
"\n",
"RED = (0, 0, 255)\n",
"BLUE = (255, 0, 0)\n",
"GREEN = (0, 255, 0)\n",
"YELLOW = (0, 255, 255)\n",
"CONTOUR_THICKNESS = 2\n",
"MAX_DISP_DIM = 500"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#a generic error\n",
"class BoggleError(Exception):\n",
" def __init__(self, arg):\n",
" self.strerror = arg\n",
" self.args = {arg}\n",
"\n",
"def four_point_transform(image, pts, size):\n",
" w = size\n",
" h = size\n",
" ntl = [0, 0]\n",
" ntr = [w, 0]\n",
" nbr = [w, h]\n",
" nbl = [0, h]\n",
" location = np.float32(pts)\n",
" x0 = pts[0][0]\n",
" x2 = pts[2][0]\n",
" #make sure the board ends up rotated the correct way\n",
" if x0 < x2:\n",
" newLocation = np.float32([ntl, nbl, nbr, ntr])\n",
" else:\n",
" newLocation = np.float32([ntr, ntl, nbl, nbr])\n",
" M = cv2.getPerspectiveTransform(location, newLocation)\n",
" warpedimage = cv2.warpPerspective(image, M, (w, h))\n",
" return warpedimage\n",
"\n",
"def resizeWithAspectRatio(image, maxDispDim, inter=cv2.INTER_AREA):\n",
" w = image.shape[0] #TODO width and height are swapped here\n",
" h = image.shape[1]\n",
"\n",
" ar = w / h\n",
" #print(ar)\n",
" if w > h:\n",
" nw = maxDispDim\n",
" nh = int(maxDispDim / ar)\n",
" elif h > w:\n",
" nh = maxDispDim\n",
" nw = int(maxDispDim / ar)\n",
" else:\n",
" nh = maxDispDim\n",
" nw = maxDispDim\n",
"\n",
" dim = None\n",
" (h, w) = image.shape[:2]\n",
"\n",
" r = nw / float(w)\n",
" dim = (nw, int(h * r))\n",
"\n",
" return cv2.resize(image, dim, interpolation=inter)\n",
"\n",
"def imshow_fit(name, img, maxDispDim=MAX_DISP_DIM):\n",
" if maxDispDim is not None:\n",
" img = resizeWithAspectRatio(img, maxDispDim)\n",
" cv2.imshow(name, img)\n",
"\n",
"def findBestCtr(contours):\n",
" bestArea = 0\n",
" bestCtr = None\n",
" for i, ctr in enumerate(contours):\n",
" area = cv2.contourArea(ctr)\n",
" # perimeter = cv2.arcLength(ctr, True)\n",
" if area > bestArea:\n",
" bestArea = area\n",
" bestCtr = ctr\n",
" return bestCtr\n",
"\n",
"\n",
"def angleEveryFew(ctr, step):\n",
" angles = []\n",
" # dists = []\n",
" prev = ctr[-1]\n",
" for i in range(0, len(ctr), step):\n",
" curr = ctr[i]\n",
" px, py = prev[0]\n",
" cx, cy = curr[0]\n",
" # angle = (px-cx)/(py-cy)\n",
" angle = math.atan2(cy - py, cx - px)\n",
" angles.append(angle)\n",
" # dist = math.hypot(cx-px, cy-py)\n",
" # dists.append(dist)\n",
" prev = curr\n",
" return angles\n",
"\n",
"\n",
"def angleAvg(angles):\n",
" x = 0\n",
" y = 0\n",
" for angle in angles:\n",
" x += math.cos(angle)\n",
" y += math.sin(angle)\n",
" return math.atan2(y, x)\n",
"\n",
"\n",
"def angleDiffAbs(angle1, angle2):\n",
" return abs(((angle1 - angle2 + math.pi) % (2 * math.pi)) - math.pi)\n",
"\n",
"\n",
"def runningAvg(angles, history):\n",
" result = []\n",
" for i in range(len(angles)):\n",
" # avg = 0\n",
" vals2 = []\n",
" for j in range(history):\n",
" val = angles[int(i + j - history / 2) % len(angles)]\n",
" vals2.append(val)\n",
" # avg += val\n",
" # avg /= history\n",
" avg = angleAvg(vals2)\n",
" result.append(avg)\n",
" return result\n",
"\n",
"\n",
"def diffAbs(vals):\n",
" diffs = []\n",
" prev = vals[-1]\n",
" for i in range(0, len(vals), 1):\n",
" curr = vals[i]\n",
" diff = angleDiffAbs(prev, curr) * 10\n",
" diffs.append(diff)\n",
" prev = curr\n",
" return diffs\n",
"\n",
"\n",
"def debounce(bools, history):\n",
" result = []\n",
" for i in range(len(bools)):\n",
" total = 0\n",
" for j in range(history):\n",
" val = bools[int(i + j - history / 2) % len(bools)]\n",
" total += val\n",
" result.append(int(total >= history / 2))\n",
" return result\n",
"\n",
"\n",
"def findGaps(diffs):\n",
" wasGap = True\n",
" seamI = 0\n",
" for i, diff in enumerate(diffs):\n",
" isGap = diff\n",
" if not isGap and not wasGap:\n",
" seamI = i\n",
" break\n",
" wasGap = isGap\n",
"\n",
" xs = range(len(diffs))\n",
" xs = [x for x in xs]\n",
" xs_a = xs[seamI:]\n",
" xs_a.extend(xs[:seamI])\n",
" diffs_a = diffs[seamI:]\n",
" diffs_a.extend(diffs[:seamI])\n",
"\n",
" xs2 = []\n",
" diffs2 = []\n",
" gapsStart = []\n",
" gapsStartY = []\n",
" gapsEnd = []\n",
" gapsEndY = []\n",
" wasGap = None\n",
" gapwidth = 0\n",
" for x, diff in zip(xs_a, diffs_a):\n",
" isGap = diff\n",
" if wasGap is None:\n",
" wasGap = isGap\n",
" if isGap:\n",
" xs2.append(x)\n",
" diffs2.append(diff)\n",
" gapwidth += 1\n",
" if isGap and not wasGap:\n",
" gapsStart.append(x)\n",
" gapsStartY.append(.5)\n",
" if not isGap and wasGap:\n",
" gapsEnd.append(x)\n",
" gapsEndY.append(gapwidth)\n",
" gapwidth = 0\n",
" wasGap = isGap\n",
" return gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2\n",
" # gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]\n",
" # return gaps, diffs, xs2\n",
"\n",
"\n",
"def top4gaps(gaps):\n",
" def length_sort(gap):\n",
" return -gap[2]\n",
"\n",
" gaps2 = sorted(gaps, key=length_sort)\n",
" gaps2 = gaps2[:4]\n",
" return [i for i in gaps if i in gaps2]\n",
"\n",
"\n",
"def invertGaps(gaps):\n",
" segments = []\n",
" prev = gaps[-1]\n",
" for curr in gaps:\n",
" segments.append((prev[1], curr[0]))\n",
" prev = curr\n",
" return segments\n",
"\n",
"\n",
"def findSidePoints(segments, ctr, step):\n",
" sidePoints = []\n",
" for seg in segments:\n",
" if seg[1] > seg[0]:\n",
" sidePoints.append(ctr[seg[0] * step:seg[1] * step])\n",
" else:\n",
" sidePoints.append(ctr[seg[0] * step:, :seg[1] * step])\n",
" return sidePoints\n",
"\n",
"\n",
"def getEndVals(arr, fraction):\n",
" if fraction >= 0.5: return arr\n",
" l = len(arr)\n",
" keep = int(fraction * l)\n",
" keep = max(keep, 1)\n",
" return np.concatenate((arr[:keep], arr[l - keep:]))\n",
"\n",
"\n",
"def fitSidePointsToLines(sidePoints):\n",
" lines = []\n",
" for sp in sidePoints:\n",
" xs = np.zeros(len(sp), int)\n",
" ys = np.zeros(len(sp), int)\n",
" for i, pt in enumerate(sp):\n",
" x, y = pt[0] #TODO if pt is empty\n",
" xs[i] = x\n",
" ys[i] = y\n",
" lines.append(np.polyfit(xs, ys, 1))\n",
" return lines\n",
"\n",
"\n",
"def findCorners(lines):\n",
" points = []\n",
"\n",
" prev = lines[-1]\n",
" for curr in lines:\n",
" a1, b1 = prev\n",
" a2, b2 = curr\n",
"\n",
" x = (b2 - b1) / (a1 - a2)\n",
" y = np.polyval(curr, x)\n",
" #if math.isnan(x): x = 0 #TODO\n",
" #if math.isnan(y): y = 0\n",
" points.append((int(x), int(y)))\n",
" prev = curr\n",
" return points\n",
"\n",
"\n",
"def drawLinesAndPoints(image, lines, points):\n",
" width = image.shape[1]\n",
"\n",
" for l in lines:\n",
" y1 = int(np.polyval(l, 0))\n",
" y2 = int(np.polyval(l, width - 1))\n",
" cv2.line(image, (0, y1), (width - 1, y2), RED, CONTOUR_THICKNESS)\n",
"\n",
" for point in points:\n",
" cv2.circle(image, point, 4, BLUE, 3)\n",
"\n",
"#https://scipy-cookbook.readthedocs.io/items/SignalSmooth.html\n",
"#window: np.ones (flat), np.hanning, np.hamming, np.bartlett, np.blackman\n",
"def smooth(x, window_len=11, window=np.hanning):\n",
" if x.ndim != 1:\n",
" raise ValueError(\"smooth only accepts 1 dimension arrays.\")\n",
"\n",
" if x.size < window_len:\n",
" raise ValueError(\"Input vector needs to be bigger than window size.\")\n",
"\n",
" if window_len<3:\n",
" return x\n",
"\n",
" s=np.r_[x[window_len-1:0:-1], x, x[-2:-window_len-1:-1]]\n",
" w = window(window_len)\n",
" y = np.convolve(w / w.sum(), s, mode='valid')\n",
" return y\n",
"\n",
"\n",
"def findRowsOrCols(img, doCols, smoothFactor, ax):\n",
" smoothFactor = int(smoothFactor * img.shape[0])\n",
" #print(\"smoothFactor\", smoothFactor)\n",
" \n",
" if doCols:\n",
" title = \"colSum\"\n",
" imgSum = cv2.reduce(img, 0, cv2.REDUCE_AVG, dtype=cv2.CV_32S)\n",
" imgSum = imgSum[0]\n",
" else:\n",
" #row sum\n",
" title = \"rowSum\"\n",
" imgSum = cv2.reduce(img, 1, cv2.REDUCE_AVG, dtype=cv2.CV_32S)\n",
" imgSum = imgSum.reshape(len(imgSum))\n",
" \n",
" imgSumSmooth = smooth(imgSum, smoothFactor*2)\n",
"\n",
" #https://qingkaikong.blogspot.com/2018/07/find-peaks-in-data.html\n",
" #peaks_positive, _ = scipy.signal.find_peaks(imgSumSmooth, height=200, threshold = None, distance=60)\n",
" dips, props = scipy.signal.find_peaks(-imgSumSmooth, height=(None,None), distance=30, prominence=(None,None))\n",
" \n",
" #threshold=(None,None), \n",
" #, plateau_size=(None,None)\n",
" \n",
" #print(props)\n",
"\n",
" prs = props[\"prominences\"]\n",
" if len(prs) < 6:\n",
" top_6_dips = dips\n",
" #print (\"!!!! less than 6 dips\")\n",
" #raise BoggleError(\"less than 6 dips\")\n",
" else:\n",
" prsIdx = sorted(range(len(prs)), key=lambda i: prs[i], reverse=True)\n",
" #print(prsIdx)\n",
" prsIdx = prsIdx[:6]\n",
" #print(prsIdx)\n",
" top_6_dips = [p for i,p in enumerate(dips) if i in prsIdx]\n",
"\n",
" #fig = plt.figure()\n",
" #ax1 = fig.add_subplot(111)\n",
" \n",
" if ax is not None:\n",
" q = [i for i in range(len(imgSumSmooth))]\n",
" \n",
" ax.plot(q, imgSumSmooth, 'b-', linewidth=2, label=\"smooth\")\n",
" ax.plot(np.linspace(smoothFactor,len(imgSumSmooth)-smoothFactor, len(imgSum)), imgSum, 'r-', linewidth=1, label=title)\n",
"\n",
" #ax.plot(\n",
" #[q[p] for p in peaks_positive],\n",
" #[imgSumSmooth[p] for p in peaks_positive],\n",
" #'ro', label = 'positive peaks')\n",
" \n",
" ax.plot(\n",
" [q[p] for p in dips],\n",
" [imgSumSmooth[p] for p in dips],\n",
" 'go', label='dips')\n",
" \n",
" ax.plot(\n",
" [q[p] for p in top_6_dips],\n",
" [imgSumSmooth[p] for p in top_6_dips],\n",
" 'c.', label='top 6 dips')\n",
" \n",
" ax.legend(loc='best')\n",
" \n",
" #return top_6_dips\n",
" #print(\"before clip\", top_6_dips)\n",
" top_6_dips_scaled = [np.clip(0, p-smoothFactor, len(imgSum)-1) for p in top_6_dips]\n",
" return top_6_dips_scaled"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def findBoggleBoard(image, normalPlots=True, harshErrors=False, generate=(\"debugimage\", \"debugmask\", \"contourPlotImg\", \"warpedimage\", \"imgSumPlotImg\", \"diceRaw\", \"dice\")):\n",
" resultImages = {}\n",
"\n",
" # maskThresholdMin = (108, 28, 12)\n",
" # maskThresholdMax = (125, 255, 241)\n",
" # maskThresholdMin = (108, 28, 6)\n",
" # maskThresholdMax = (130, 255, 241)\n",
" maskThresholdMin = (108, 28, 6)\n",
" maskThresholdMax = (144, 255, 241)\n",
" size = max(image.shape)\n",
" #print(\"size\", size)\n",
" blurAmount = int(.02 * size)\n",
" blurAmount = (blurAmount, blurAmount)\n",
" # blurThreshold = 80\n",
" blurThreshold = 40\n",
" contourApprox = cv2.CHAIN_APPROX_NONE\n",
" # contourApprox = cv2.CHAIN_APPROX_SIMPLE\n",
" # contourApprox = cv2.CHAIN_APPROX_TC89_L1\n",
" # contourApprox = cv2.CHAIN_APPROX_TC89_KCOS\n",
"\n",
" hsvimg = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n",
" mask = cv2.inRange(hsvimg, maskThresholdMin, maskThresholdMax)\n",
"\n",
" maskblur = cv2.blur(mask, blurAmount)\n",
" # maskblur = cv2.threshold(maskblur, 80, 255, cv2.THRESH_BINARY_INV)\n",
" maskblur = cv2.inRange(maskblur, (blurThreshold,), (255,))\n",
" \n",
" #API CHANGE: findContours no longer returns a modified image\n",
" #contourimg, contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)\n",
" contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)\n",
" # print(hierarchy) #TODO\n",
" \n",
" bestCtr = findBestCtr(contours)\n",
" # bestCtr = cv2.convexHull(bestCtr)\n",
"\n",
" #draw the contours for debugging\n",
" if \"debugimage\" in generate:\n",
" debugimage = image.copy()\n",
" cv2.drawContours(debugimage, contours, -1, RED, CONTOUR_THICKNESS)\n",
" cv2.drawContours(debugimage, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)\n",
" if \"debugmask\" in generate:\n",
" debugmask = cv2.cvtColor(maskblur, cv2.COLOR_GRAY2BGR)\n",
" cv2.drawContours(debugmask, contours, -1, RED, CONTOUR_THICKNESS)\n",
" cv2.drawContours(debugmask, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)\n",
" \n",
"\n",
" step = 10\n",
" avgWindow = 0.1\n",
" debounceFactor = .05\n",
" \n",
" angles = angleEveryFew(bestCtr, step)\n",
"\n",
" xs = range(len(angles))\n",
"\n",
" anglesAvg = runningAvg(angles, int(avgWindow * len(angles)))\n",
" diffs = diffAbs(anglesAvg)\n",
" \n",
" avgDiff = np.mean(diffs)\n",
" #print(avgDiff)\n",
" \n",
" binDiffs = [int(i > avgDiff) for i in diffs]\n",
" binDiffs = debounce(binDiffs, int(len(binDiffs) * debounceFactor))\n",
" gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2 = findGaps(binDiffs)\n",
" gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]\n",
" \n",
" #scale for viewing on the plot\n",
" gapsEndY2 = gapsEndY\n",
" if len(gapsEndY) > 0:\n",
" q = max(gapsEndY)\n",
" if q > 6: gapsEndY2 = [i * 6 / q for i in gapsEndY]\n",
"\n",
" if \"contourPlotImg\" in generate:\n",
" contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, gapsEndY2, normalPlots)\n",
" if contourPlotImg is not None:\n",
" resultImages[\"contourPlotImg\"] = contourPlotImg\n",
" \n",
" if len(gaps) < 4:\n",
" print(\"!!!! less than 4 gaps\")\n",
" if harshErrors:\n",
" raise BoggleError(\"less than 4 gaps\")\n",
" if \"contourPlotImg\" in generate:\n",
" contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, None, normalPlots)\n",
" if contourPlotImg is not None:\n",
" resultImages[\"contourPlotImg\"] = contourPlotImg\n",
" \n",
" if \"debugmask\" in generate:\n",
" resultImages[\"debugmask\"] = debugmask\n",
" if \"debugimage\" in generate:\n",
" resultImages[\"debugimage\"] = debugimage\n",
" return resultImages, None\n",
" \n",
" endFraction = 0.01\n",
" \n",
" gaps = top4gaps(gaps)\n",
" segments = invertGaps(gaps)\n",
" sidePoints = findSidePoints(segments, bestCtr, step)\n",
" sidePoints = [getEndVals(sp, endFraction) for sp in sidePoints]\n",
" #print(\"sidepoints len\", len(sidePoints[0]))\n",
" \n",
" \n",
" lines = fitSidePointsToLines(sidePoints)\n",
" points = findCorners(lines)\n",
" if \"debugimage\" in generate:\n",
" cv2.drawContours(debugimage, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)\n",
" drawLinesAndPoints(debugimage, lines, points)\n",
" resultImages[\"debugimage\"] = debugimage\n",
" if \"debugmask\" in generate:\n",
" cv2.drawContours(debugmask, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)\n",
" drawLinesAndPoints(debugmask, lines, points)\n",
" resultImages[\"debugmask\"] = debugmask\n",
"\n",
" npPoints = np.array(points)\n",
" size = 300\n",
" warpedimage = four_point_transform(image, npPoints, size)\n",
" warpgray = cv2.cvtColor(warpedimage, cv2.COLOR_BGR2GRAY)\n",
" \n",
" if \"warpedimage\" in generate:\n",
" resultImages[\"warpedimage\"] = warpedimage\n",
" \n",
" if \"warpgray\" in generate:\n",
" resultImages[\"warpgray\"] = warpgray\n",
" \n",
" smoothFactor = .05\n",
" \n",
" if \"imgSumPlotImg\" in generate:\n",
" fig, (ax0, ax1) = plt.subplots(2, figsize=(8,10))\n",
" else:\n",
" ax0 = ax1 = None\n",
" \n",
" rowSumLines = findRowsOrCols(warpgray, False, smoothFactor, ax0)\n",
" #print(\"rows\", rowSumLines)\n",
" colSumLines = findRowsOrCols(warpgray, True, smoothFactor, ax1)\n",
" #print(\"cols\", colSumLines)\n",
" \n",
" \n",
" if \"imgSumPlotImg\" in generate:\n",
" if normalPlots:\n",
" plt.show(block=False)\n",
" else:\n",
" resultImages[\"imgSumPlotImg\"] = plotToImg()\n",
"\n",
"\n",
" if len(rowSumLines) < 6 or len(colSumLines) < 6:\n",
" print(\"!!!! not enough grid lines\")\n",
" if harshErrors:\n",
" raise BoggleError(\"not enough gridlines\")\n",
" return resultImages, None\n",
" \n",
" #fix the outermost lines of the board\n",
" h1 = rowSumLines[2] - rowSumLines[1]\n",
" h2 = rowSumLines[3] - rowSumLines[2]\n",
" h3 = rowSumLines[4] - rowSumLines[3]\n",
" h = max(h1, h2, h3)\n",
" \n",
" newCSL0 = colSumLines[1] - h\n",
" if newCSL0 > colSumLines[0]:\n",
" colSumLines[0] = newCSL0\n",
" newCSL5 = colSumLines[4] + h\n",
" if newCSL5 < colSumLines[5]:\n",
" colSumLines[5] = newCSL5\n",
" \n",
" w1 = colSumLines[2] - colSumLines[1]\n",
" w2 = colSumLines[3] - colSumLines[2]\n",
" w3 = colSumLines[4] - colSumLines[3]\n",
" w = max(w1, w2, w3)\n",
" \n",
" newRSL0 = rowSumLines[1] - w\n",
" if newRSL0 > rowSumLines[0]:\n",
" rowSumLines[0] = newRSL0\n",
" newRSL5 = rowSumLines[4] + w\n",
" if newRSL5 < rowSumLines[5]:\n",
" rowSumLines[5] = newRSL5\n",
"\n",
" #print(\"rows2\", rowSumLines)\n",
" #print(\"cols2\", colSumLines)\n",
"\n",
" #just display\n",
" if \"diceRaw\" in generate:\n",
" plt.figure(figsize=(10,10))\n",
" i = 1\n",
" for y in range(5):\n",
" for x in range(5):\n",
" plt.subplot(5,5,i)\n",
" i += 1\n",
" plt.xticks([])\n",
" plt.yticks([])\n",
" plt.grid(False)\n",
" minX = colSumLines[x]\n",
" maxX = colSumLines[x+1]\n",
" minY = rowSumLines[y]\n",
" maxY = rowSumLines[y+1]\n",
" crop_img = warpgray[minY:maxY, minX:maxX]\n",
" plt.imshow(crop_img, cmap=plt.cm.gray)\n",
" if normalPlots:\n",
" plt.show(block=False)\n",
" else:\n",
" resultImages[\"diceRaw\"] = plotToImg()\n",
"\n",
" \n",
" if \"dice\" in generate:\n",
" plt.figure(figsize=(10,10))\n",
" i = 1\n",
"\n",
" letterResize = 30\n",
" #make square, resize, display, and save to an array\n",
" letterImgs = []\n",
" for y in range(5):\n",
" letterImgRow = []\n",
" for x in range(5):\n",
" minX = colSumLines[x]\n",
" maxX = colSumLines[x+1]\n",
" minY = rowSumLines[y]\n",
" maxY = rowSumLines[y+1]\n",
" w = maxX - minX\n",
" h = maxY - minY\n",
" #print(\"w,h 1:\", w, h)\n",
" d = abs(w - h)\n",
" if d > 0:\n",
" if int(d/2) == d/2:\n",
" #even difference\n",
" d1 = d2 = int(d/2)\n",
" else:\n",
" #odd difference\n",
" d1 = int((d-1)/2)\n",
" d2 = int((d+1)/2)\n",
" if w > h:\n",
" #wider than it is tall\n",
" minX += d1\n",
" maxX -= d2\n",
" else:\n",
" #taller than it is wide\n",
" minY += d1\n",
" maxY -= d2\n",
" #print(\"w,h 2:\", maxX-minX, maxY-minY)\n",
" crop_img = warpgray[minY:maxY, minX:maxX]\n",
" letterImg = cv2.resize(crop_img, (letterResize,letterResize), interpolation=cv2.INTER_AREA)\n",
" if \"dice\" in generate:\n",
" plt.subplot(5,5,i)\n",
" i += 1\n",
" plt.xticks([])\n",
" plt.yticks([])\n",
" plt.grid(False)\n",
" plt.imshow(letterImg, cmap=plt.cm.gray)\n",
" letterImgRow.append(letterImg)\n",
" letterImgs.append(letterImgRow)\n",
"\n",
" if \"dice\" in generate:\n",
" if normalPlots:\n",
" plt.show(block=False)\n",
" else:\n",
" resultImages[\"dice\"] = plotToImg()\n",
" \n",
" return resultImages, letterImgs"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#https://www.tensorflow.org/tutorials/keras/classification\n",
"\n",
"# from __future__ import absolute_import, division, print_function, unicode_literals\n",
"\n",
"# TensorFlow and tf.keras\n",
"import tensorflow as tf\n",
"from tensorflow import keras\n",
"from tensorflow.keras import datasets, layers, models\n",
"\n",
"print(tf.__version__)\n",
"\n",
"# Helper libraries\n",
"import numpy as np\n",
"# import matplotlib.pyplot as plt\n",
"\n",
"import json\n",
"\n",
"MODEL_FILE=\"/home/johanv/nextcloud/projects/boggle2.0/model.h5\"\n",
"#MODEL_URL = \"https://drive.confuzer.cloud/index.php/s/HWSczZqSjbPqKDq/download\"\n",
"#import os\n",
"#os.system(\"curl \" + MODEL_URL + \" > \" + MODEL_FILE)\n",
"\n",
"IMG_DIM = 30\n",
"\n",
"class_names = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n",
"\n",
"L = len(class_names)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = tf.keras.models.load_model(MODEL_FILE)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def processImage(filepath):\n",
" image = cv2.imread(filepath)\n",
" _, letters5x5grid = findBoggleBoard(image, normalPlots=False, harshErrors=True, generate=())\n",
" \n",
" lettersGuessed = \"\"\n",
" confidence = []\n",
" num_rows = 5\n",
" num_cols = 5\n",
" num_images = num_rows*num_cols\n",
" for row in range(num_rows):\n",
" for col in range(num_cols):\n",
" letterImg = letters5x5grid[row][col]\n",
" letterImg = letterImg / 255\n",
"# print(letterImg.shape)\n",
" letterImg = (np.expand_dims(letterImg,0))\n",
" letterImg = (np.expand_dims(letterImg,axis=3))\n",
"# print(letterImg.shape)\n",
" pred = model.predict(letterImg)[0]\n",
" letter = class_names[np.argmax(pred)]\n",
" lettersGuessed += letter\n",
" confidence.append(np.max(pred))\n",
" return lettersGuessed, confidence"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"# letters5x5gridLabelsStr = \"IZEIMFLTYTOSEINHETNORRISU\" #00162\n",
"# letters5x5gridLabelsStr = \"DONLIEEIIESAPYWTAAKLTINRE\" #00164\n",
"# letters5x5gridLabelsStr = \"RAOCODSEERGAPEWORXRHELWNT\" #00165\n",
"# letters5x5gridLabelsStr = \"RONLICTSSDNMPPNUAEQIHINRM\" #00170\n",
"letters5x5gridLabelsStr = \"VANUIRAUHESKEITTDPRCGOUCA\" #00171, 00160\n",
"# letters5x5gridLabelsStr = \"IXESMFLEYTOSEENOETNRRRIWM\" #00172, 00161\n",
"\n",
"#process 1 image\n",
"IMAGE_FILE = '/home/johanv/nextcloud/projects/boggle2.0/cascademan/categories/5x5/images/00160.jpg'\n",
"lettersGuessed, confidence = processImage(IMAGE_FILE)\n",
"\n",
"# letters5x5gridLabels = [class_names.index(letter) for letter in letters5x5gridLabelsStr]\n",
"right = \"\".join([str(int(a == b)) for a,b in zip(lettersGuessed, letters5x5gridLabelsStr)])\n",
"\n",
"confidence_right = []\n",
"confidence_wrong = []\n",
"\n",
"for i, c in enumerate(confidence):\n",
" if right[i] == \"1\":\n",
" confidence_right.append(c)\n",
" else:\n",
" confidence_wrong.append(c)\n",
"\n",
"print(\"guess: \" + lettersGuessed)\n",
"print(\"real: \" + letters5x5gridLabelsStr)\n",
"print(\"right: \" + right)\n",
"# print(\"confidence:\", confidence)\n",
"# print(\"confidence_right:\", confidence_right)\n",
"# print(\"confidence_wrong:\", confidence_wrong)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}