johanvandegriff
/
BoggleCV
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Using OpenCV and Tensorflow to identify a Boggle board in an image and decode what letters are on the board.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
29 Commits
2 Branches
0 Tags
15 MiB
 
 
Branch: master
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'master'
${ noResults }
BoggleCV/full-pipeline.ipynb

1136 lines
39 KiB
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import cv2, math, os, json, traceback, io, time\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import scipy.signal\n",
"\n",
"print(cv2.__version__)\n",
"\n",
"IMAGE_DIR = '/home/johanv/nextcloud/projects/boggle2.0/cascademan/categories/5x5/images'\n",
"IMAGE_FILE = '00170.jpg'\n",
"\n",
"RED = (0, 0, 255)\n",
"BLUE = (255, 0, 0)\n",
"GREEN = (0, 255, 0)\n",
"YELLOW = (0, 255, 255)\n",
"CONTOUR_THICKNESS = 2\n",
"MAX_DISP_DIM = 500"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#a generic error\n",
"class BoggleError(Exception):\n",
" def __init__(self, arg):\n",
" self.strerror = arg\n",
" self.args = {arg}\n",
"\n",
"def four_point_transform(image, pts, size):\n",
" w = size\n",
" h = size\n",
" ntl = [0, 0]\n",
" ntr = [w, 0]\n",
" nbr = [w, h]\n",
" nbl = [0, h]\n",
" location = np.float32(pts)\n",
" x0 = pts[0][0]\n",
" x2 = pts[2][0]\n",
" #make sure the board ends up rotated the correct way\n",
" if x0 < x2:\n",
" newLocation = np.float32([ntl, nbl, nbr, ntr])\n",
" else:\n",
" newLocation = np.float32([ntr, ntl, nbl, nbr])\n",
" M = cv2.getPerspectiveTransform(location, newLocation)\n",
" warpedimage = cv2.warpPerspective(image, M, (w, h))\n",
" return warpedimage\n",
"\n",
"def resizeWithAspectRatio(image, maxDispDim, inter=cv2.INTER_AREA):\n",
" w = image.shape[0] #TODO width and height are swapped here\n",
" h = image.shape[1]\n",
"\n",
" ar = w / h\n",
" #print(ar)\n",
" if w > h:\n",
" nw = maxDispDim\n",
" nh = int(maxDispDim / ar)\n",
" elif h > w:\n",
" nh = maxDispDim\n",
" nw = int(maxDispDim / ar)\n",
" else:\n",
" nh = maxDispDim\n",
" nw = maxDispDim\n",
"\n",
" dim = None\n",
" (h, w) = image.shape[:2]\n",
"\n",
" r = nw / float(w)\n",
" dim = (nw, int(h * r))\n",
"\n",
" return cv2.resize(image, dim, interpolation=inter)\n",
"\n",
"def imshow_fit(name, img, maxDispDim=MAX_DISP_DIM):\n",
" if maxDispDim is not None:\n",
" img = resizeWithAspectRatio(img, maxDispDim)\n",
" cv2.imshow(name, img)\n",
"\n",
"def findBestCtr(contours):\n",
" bestArea = 0\n",
" bestCtr = None\n",
" for i, ctr in enumerate(contours):\n",
" area = cv2.contourArea(ctr)\n",
" # perimeter = cv2.arcLength(ctr, True)\n",
" if area > bestArea:\n",
" bestArea = area\n",
" bestCtr = ctr\n",
" return bestCtr\n",
"\n",
"\n",
"def angleEveryFew(ctr, step):\n",
" angles = []\n",
" # dists = []\n",
" prev = ctr[-1]\n",
" for i in range(0, len(ctr), step):\n",
" curr = ctr[i]\n",
" px, py = prev[0]\n",
" cx, cy = curr[0]\n",
" # angle = (px-cx)/(py-cy)\n",
" angle = math.atan2(cy - py, cx - px)\n",
" angles.append(angle)\n",
" # dist = math.hypot(cx-px, cy-py)\n",
" # dists.append(dist)\n",
" prev = curr\n",
" return angles\n",
"\n",
"\n",
"def angleAvg(angles):\n",
" x = 0\n",
" y = 0\n",
" for angle in angles:\n",
" x += math.cos(angle)\n",
" y += math.sin(angle)\n",
" return math.atan2(y, x)\n",
"\n",
"\n",
"def angleDiffAbs(angle1, angle2):\n",
" return abs(((angle1 - angle2 + math.pi) % (2 * math.pi)) - math.pi)\n",
"\n",
"\n",
"def runningAvg(angles, history):\n",
" result = []\n",
" for i in range(len(angles)):\n",
" # avg = 0\n",
" vals2 = []\n",
" for j in range(history):\n",
" val = angles[int(i + j - history / 2) % len(angles)]\n",
" vals2.append(val)\n",
" # avg += val\n",
" # avg /= history\n",
" avg = angleAvg(vals2)\n",
" result.append(avg)\n",
" return result\n",
"\n",
"\n",
"def diffAbs(vals):\n",
" diffs = []\n",
" prev = vals[-1]\n",
" for i in range(0, len(vals), 1):\n",
" curr = vals[i]\n",
" diff = angleDiffAbs(prev, curr) * 10\n",
" diffs.append(diff)\n",
" prev = curr\n",
" return diffs\n",
"\n",
"\n",
"def debounce(bools, history):\n",
" result = []\n",
" for i in range(len(bools)):\n",
" total = 0\n",
" for j in range(history):\n",
" val = bools[int(i + j - history / 2) % len(bools)]\n",
" total += val\n",
" result.append(int(total >= history / 2))\n",
" return result\n",
"\n",
"\n",
"def findGaps(diffs):\n",
" wasGap = True\n",
" seamI = 0\n",
" for i, diff in enumerate(diffs):\n",
" isGap = diff\n",
" if not isGap and not wasGap:\n",
" seamI = i\n",
" break\n",
" wasGap = isGap\n",
"\n",
" xs = range(len(diffs))\n",
" xs = [x for x in xs]\n",
" xs_a = xs[seamI:]\n",
" xs_a.extend(xs[:seamI])\n",
" diffs_a = diffs[seamI:]\n",
" diffs_a.extend(diffs[:seamI])\n",
"\n",
" xs2 = []\n",
" diffs2 = []\n",
" gapsStart = []\n",
" gapsStartY = []\n",
" gapsEnd = []\n",
" gapsEndY = []\n",
" wasGap = None\n",
" gapwidth = 0\n",
" for x, diff in zip(xs_a, diffs_a):\n",
" isGap = diff\n",
" if wasGap is None:\n",
" wasGap = isGap\n",
" if isGap:\n",
" xs2.append(x)\n",
" diffs2.append(diff)\n",
" gapwidth += 1\n",
" if isGap and not wasGap:\n",
" gapsStart.append(x)\n",
" gapsStartY.append(.5)\n",
" if not isGap and wasGap:\n",
" gapsEnd.append(x)\n",
" gapsEndY.append(gapwidth)\n",
" gapwidth = 0\n",
" wasGap = isGap\n",
" return gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2\n",
" # gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]\n",
" # return gaps, diffs, xs2\n",
"\n",
"\n",
"def top4gaps(gaps):\n",
" def length_sort(gap):\n",
" return -gap[2]\n",
"\n",
" gaps2 = sorted(gaps, key=length_sort)\n",
" gaps2 = gaps2[:4]\n",
" return [i for i in gaps if i in gaps2]\n",
"\n",
"\n",
"def invertGaps(gaps):\n",
" segments = []\n",
" prev = gaps[-1]\n",
" for curr in gaps:\n",
" segments.append((prev[1], curr[0]))\n",
" prev = curr\n",
" return segments\n",
"\n",
"\n",
"def findSidePoints(segments, ctr, step):\n",
" sidePoints = []\n",
" for seg in segments:\n",
" if seg[1] > seg[0]:\n",
" sidePoints.append(ctr[seg[0] * step:seg[1] * step])\n",
" else:\n",
" sidePoints.append(ctr[seg[0] * step:, :seg[1] * step])\n",
" return sidePoints\n",
"\n",
"\n",
"def getEndVals(arr, fraction):\n",
" if fraction >= 0.5: return arr\n",
" l = len(arr)\n",
" keep = int(fraction * l)\n",
" keep = max(keep, 1)\n",
" return np.concatenate((arr[:keep], arr[l - keep:]))\n",
"\n",
"\n",
"def fitSidePointsToLines(sidePoints):\n",
" lines = []\n",
" for sp in sidePoints:\n",
" xs = np.zeros(len(sp), int)\n",
" ys = np.zeros(len(sp), int)\n",
" for i, pt in enumerate(sp):\n",
" x, y = pt[0] #TODO if pt is empty\n",
" xs[i] = x\n",
" ys[i] = y\n",
" lines.append(np.polyfit(xs, ys, 1))\n",
" return lines\n",
"\n",
"\n",
"def findCorners(lines):\n",
" points = []\n",
"\n",
" prev = lines[-1]\n",
" for curr in lines:\n",
" a1, b1 = prev\n",
" a2, b2 = curr\n",
"\n",
" x = (b2 - b1) / (a1 - a2)\n",
" y = np.polyval(curr, x)\n",
" #if math.isnan(x): x = 0 #TODO\n",
" #if math.isnan(y): y = 0\n",
" points.append((int(x), int(y)))\n",
" prev = curr\n",
" return points\n",
"\n",
"\n",
"def drawLinesAndPoints(image, lines, points):\n",
" width = image.shape[1]\n",
"\n",
" for l in lines:\n",
" y1 = int(np.polyval(l, 0))\n",
" y2 = int(np.polyval(l, width - 1))\n",
" cv2.line(image, (0, y1), (width - 1, y2), RED, CONTOUR_THICKNESS)\n",
"\n",
" for point in points:\n",
" cv2.circle(image, point, 4, BLUE, 3)\n",
"\n",
"\n",
"def contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, gapsEndY2, normalPlots):\n",
" fig = plt.figure(figsize=(8,10))\n",
" ax1 = fig.add_subplot(111)\n",
"\n",
" # ax1.scatter(xs, dists, s=10, c='b', marker=\"s\", label=\"dists\")\n",
" ax1.scatter(xs, angles, s=10, c='r', marker=\"o\", label=\"angles\")\n",
" ax1.scatter(xs, anglesAvg, s=10, c='b', marker=\"o\", label=\"anglesAvg\")\n",
" ax1.scatter(xs, diffs, s=10, c='g', marker=\"o\", label=\"diffs\")\n",
" ax1.scatter(xs2, diffs2, s=10, c='m', marker=\"s\", label=\"diffs2\")\n",
" ax1.scatter(gapsStart, gapsStartY, s=10, c='c', marker=\"o\", label=\"gapsStart\")\n",
" if gapsEndY2 is not None:\n",
" ax1.scatter(gapsEnd, gapsEndY2, s=10, c='y', marker=\"o\", label=\"gapsEnd\")\n",
" plt.legend(loc='best')\n",
" if normalPlots:\n",
" plt.show(block=False)\n",
" return None\n",
" else:\n",
" return plotToImg()\n",
"\n",
"def waitForKey():\n",
" while True:\n",
" key = cv2.waitKey(0)\n",
" print(\"key\", key)\n",
" if key == 27: # esc\n",
" cv2.destroyAllWindows()\n",
" quit()\n",
" if key == ord(\" \") or key == ord(\"q\"):\n",
" break\n",
" cv2.destroyAllWindows()\n",
" plt.close('all')\n",
"\n",
"def waitForConsoleEnter():\n",
" print(\"=== hit enter to continue ===\")\n",
" input()\n",
" print(\"=== continuing ===\")\n",
" cv2.destroyAllWindows()\n",
" plt.close('all')\n",
"\n",
"#https://scipy-cookbook.readthedocs.io/items/SignalSmooth.html\n",
"#window: np.ones (flat), np.hanning, np.hamming, np.bartlett, np.blackman\n",
"def smooth(x, window_len=11, window=np.hanning):\n",
" if x.ndim != 1:\n",
" raise ValueError(\"smooth only accepts 1 dimension arrays.\")\n",
"\n",
" if x.size < window_len:\n",
" raise ValueError(\"Input vector needs to be bigger than window size.\")\n",
"\n",
" if window_len<3:\n",
" return x\n",
"\n",
" s=np.r_[x[window_len-1:0:-1], x, x[-2:-window_len-1:-1]]\n",
" w = window(window_len)\n",
" y = np.convolve(w / w.sum(), s, mode='valid')\n",
" return y\n",
"\n",
"\n",
"def findRowsOrCols(img, doCols, smoothFactor, ax):\n",
" smoothFactor = int(smoothFactor * img.shape[0])\n",
" #print(\"smoothFactor\", smoothFactor)\n",
" \n",
" if doCols:\n",
" title = \"colSum\"\n",
" imgSum = cv2.reduce(img, 0, cv2.REDUCE_AVG, dtype=cv2.CV_32S)\n",
" imgSum = imgSum[0]\n",
" else:\n",
" #row sum\n",
" title = \"rowSum\"\n",
" imgSum = cv2.reduce(img, 1, cv2.REDUCE_AVG, dtype=cv2.CV_32S)\n",
" imgSum = imgSum.reshape(len(imgSum))\n",
" \n",
" imgSumSmooth = smooth(imgSum, smoothFactor*2)\n",
"\n",
" #https://qingkaikong.blogspot.com/2018/07/find-peaks-in-data.html\n",
" #peaks_positive, _ = scipy.signal.find_peaks(imgSumSmooth, height=200, threshold = None, distance=60)\n",
" dips, props = scipy.signal.find_peaks(-imgSumSmooth, height=(None,None), distance=30, prominence=(None,None))\n",
" \n",
" #threshold=(None,None), \n",
" #, plateau_size=(None,None)\n",
" \n",
" #print(props)\n",
"\n",
" prs = props[\"prominences\"]\n",
" if len(prs) < 6:\n",
" top_6_dips = dips\n",
" #print (\"!!!! less than 6 dips\")\n",
" #raise BoggleError(\"less than 6 dips\")\n",
" else:\n",
" prsIdx = sorted(range(len(prs)), key=lambda i: prs[i], reverse=True)\n",
" #print(prsIdx)\n",
" prsIdx = prsIdx[:6]\n",
" #print(prsIdx)\n",
" top_6_dips = [p for i,p in enumerate(dips) if i in prsIdx]\n",
"\n",
" #fig = plt.figure()\n",
" #ax1 = fig.add_subplot(111)\n",
" \n",
" if ax is not None:\n",
" q = [i for i in range(len(imgSumSmooth))]\n",
" \n",
" ax.plot(q, imgSumSmooth, 'b-', linewidth=2, label=\"smooth\")\n",
" ax.plot(np.linspace(smoothFactor,len(imgSumSmooth)-smoothFactor, len(imgSum)), imgSum, 'r-', linewidth=1, label=title)\n",
"\n",
" #ax.plot(\n",
" #[q[p] for p in peaks_positive],\n",
" #[imgSumSmooth[p] for p in peaks_positive],\n",
" #'ro', label = 'positive peaks')\n",
" \n",
" ax.plot(\n",
" [q[p] for p in dips],\n",
" [imgSumSmooth[p] for p in dips],\n",
" 'go', label='dips')\n",
" \n",
" ax.plot(\n",
" [q[p] for p in top_6_dips],\n",
" [imgSumSmooth[p] for p in top_6_dips],\n",
" 'c.', label='top 6 dips')\n",
" \n",
" ax.legend(loc='best')\n",
" \n",
" #return top_6_dips\n",
" #print(\"before clip\", top_6_dips)\n",
" top_6_dips_scaled = [np.clip(0, p-smoothFactor, len(imgSum)-1) for p in top_6_dips]\n",
" return top_6_dips_scaled\n",
"\n",
"def plotToImg():\n",
" #https://stackoverflow.com/questions/5314707/matplotlib-store-image-in-variable\n",
" buf = io.BytesIO()\n",
" plt.savefig(buf, format='png', bbox_inches='tight')\n",
" plt.close()\n",
" buf.seek(0)\n",
" \n",
" #https://stackoverflow.com/questions/11552926/how-to-read-raw-png-from-an-array-in-python-opencv\n",
" file_bytes = np.asarray(bytearray(buf.read()), dtype=np.uint8)\n",
" img_data_ndarray = cv2.imdecode(file_bytes, cv2.IMREAD_UNCHANGED)\n",
" #img_data_cvmat = cv.fromarray(img_data_ndarray) # convert to old cvmat if needed\n",
"\n",
" img_rgb = cv2.cvtColor(img_data_ndarray, cv2.COLOR_RGBA2RGB)\n",
" return img_rgb"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def findBoggleBoard(image, normalPlots=True, harshErrors=False, generate=(\"debugimage\", \"debugmask\", \"contourPlotImg\", \"warpedimage\", \"imgSumPlotImg\", \"diceRaw\", \"dice\")):\n",
" resultImages = {}\n",
"\n",
" # maskThresholdMin = (108, 28, 12)\n",
" # maskThresholdMax = (125, 255, 241)\n",
" # maskThresholdMin = (108, 28, 6)\n",
" # maskThresholdMax = (130, 255, 241)\n",
" maskThresholdMin = (108, 28, 6)\n",
" maskThresholdMax = (144, 255, 241)\n",
" size = max(image.shape)\n",
" #print(\"size\", size)\n",
" blurAmount = int(.02 * size)\n",
" blurAmount = (blurAmount, blurAmount)\n",
" # blurThreshold = 80\n",
" blurThreshold = 40\n",
" contourApprox = cv2.CHAIN_APPROX_NONE\n",
" # contourApprox = cv2.CHAIN_APPROX_SIMPLE\n",
" # contourApprox = cv2.CHAIN_APPROX_TC89_L1\n",
" # contourApprox = cv2.CHAIN_APPROX_TC89_KCOS\n",
"\n",
" hsvimg = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)\n",
" mask = cv2.inRange(hsvimg, maskThresholdMin, maskThresholdMax)\n",
"\n",
" maskblur = cv2.blur(mask, blurAmount)\n",
" # maskblur = cv2.threshold(maskblur, 80, 255, cv2.THRESH_BINARY_INV)\n",
" maskblur = cv2.inRange(maskblur, (blurThreshold,), (255,))\n",
" \n",
" #API CHANGE: findContours no longer returns a modified image\n",
" #contourimg, contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)\n",
" contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)\n",
" # print(hierarchy) #TODO\n",
" \n",
" bestCtr = findBestCtr(contours)\n",
" # bestCtr = cv2.convexHull(bestCtr)\n",
"\n",
" #draw the contours for debugging\n",
" if \"debugimage\" in generate:\n",
" debugimage = image.copy()\n",
" cv2.drawContours(debugimage, contours, -1, RED, CONTOUR_THICKNESS)\n",
" cv2.drawContours(debugimage, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)\n",
" if \"debugmask\" in generate:\n",
" debugmask = cv2.cvtColor(maskblur, cv2.COLOR_GRAY2BGR)\n",
" cv2.drawContours(debugmask, contours, -1, RED, CONTOUR_THICKNESS)\n",
" cv2.drawContours(debugmask, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)\n",
" \n",
"\n",
" step = 10\n",
" avgWindow = 0.1\n",
" debounceFactor = .05\n",
" \n",
" angles = angleEveryFew(bestCtr, step)\n",
"\n",
" xs = range(len(angles))\n",
"\n",
" anglesAvg = runningAvg(angles, int(avgWindow * len(angles)))\n",
" diffs = diffAbs(anglesAvg)\n",
" \n",
" avgDiff = np.mean(diffs)\n",
" #print(avgDiff)\n",
" \n",
" binDiffs = [int(i > avgDiff) for i in diffs]\n",
" binDiffs = debounce(binDiffs, int(len(binDiffs) * debounceFactor))\n",
" gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2 = findGaps(binDiffs)\n",
" gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]\n",
" \n",
" #scale for viewing on the plot\n",
" gapsEndY2 = gapsEndY\n",
" if len(gapsEndY) > 0:\n",
" q = max(gapsEndY)\n",
" if q > 6: gapsEndY2 = [i * 6 / q for i in gapsEndY]\n",
"\n",
" if \"contourPlotImg\" in generate:\n",
" contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, gapsEndY2, normalPlots)\n",
" if contourPlotImg is not None:\n",
" resultImages[\"contourPlotImg\"] = contourPlotImg\n",
" \n",
" if len(gaps) < 4:\n",
" print(\"!!!! less than 4 gaps\")\n",
" if harshErrors:\n",
" raise BoggleError(\"less than 4 gaps\")\n",
" if \"contourPlotImg\" in generate:\n",
" contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, None, normalPlots)\n",
" if contourPlotImg is not None:\n",
" resultImages[\"contourPlotImg\"] = contourPlotImg\n",
" \n",
" if \"debugmask\" in generate:\n",
" resultImages[\"debugmask\"] = debugmask\n",
" if \"debugimage\" in generate:\n",
" resultImages[\"debugimage\"] = debugimage\n",
" return resultImages, None\n",
" \n",
" endFraction = 0.01\n",
" \n",
" gaps = top4gaps(gaps)\n",
" segments = invertGaps(gaps)\n",
" sidePoints = findSidePoints(segments, bestCtr, step)\n",
" sidePoints = [getEndVals(sp, endFraction) for sp in sidePoints]\n",
" #print(\"sidepoints len\", len(sidePoints[0]))\n",
" \n",
" \n",
" lines = fitSidePointsToLines(sidePoints)\n",
" points = findCorners(lines)\n",
" if \"debugimage\" in generate:\n",
" cv2.drawContours(debugimage, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)\n",
" drawLinesAndPoints(debugimage, lines, points)\n",
" resultImages[\"debugimage\"] = debugimage\n",
" if \"debugmask\" in generate:\n",
" cv2.drawContours(debugmask, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)\n",
" drawLinesAndPoints(debugmask, lines, points)\n",
" resultImages[\"debugmask\"] = debugmask\n",
"\n",
" npPoints = np.array(points)\n",
" size = 300\n",
" warpedimage = four_point_transform(image, npPoints, size)\n",
" warpgray = cv2.cvtColor(warpedimage, cv2.COLOR_BGR2GRAY)\n",
" \n",
" if \"warpedimage\" in generate:\n",
" resultImages[\"warpedimage\"] = warpedimage\n",
" \n",
" if \"warpgray\" in generate:\n",
" resultImages[\"warpgray\"] = warpgray\n",
" \n",
" smoothFactor = .05\n",
" \n",
" if \"imgSumPlotImg\" in generate:\n",
" fig, (ax0, ax1) = plt.subplots(2, figsize=(8,10))\n",
" else:\n",
" ax0 = ax1 = None\n",
" \n",
" rowSumLines = findRowsOrCols(warpgray, False, smoothFactor, ax0)\n",
" #print(\"rows\", rowSumLines)\n",
" colSumLines = findRowsOrCols(warpgray, True, smoothFactor, ax1)\n",
" #print(\"cols\", colSumLines)\n",
" \n",
" \n",
" if \"imgSumPlotImg\" in generate:\n",
" if normalPlots:\n",
" plt.show(block=False)\n",
" else:\n",
" resultImages[\"imgSumPlotImg\"] = plotToImg()\n",
"\n",
"\n",
" if len(rowSumLines) < 6 or len(colSumLines) < 6:\n",
" print(\"!!!! not enough grid lines\")\n",
" if harshErrors:\n",
" raise BoggleError(\"not enough gridlines\")\n",
" return resultImages, None\n",
" \n",
" #fix the outermost lines of the board\n",
" h1 = rowSumLines[2] - rowSumLines[1]\n",
" h2 = rowSumLines[3] - rowSumLines[2]\n",
" h3 = rowSumLines[4] - rowSumLines[3]\n",
" h = max(h1, h2, h3)\n",
" \n",
" newCSL0 = colSumLines[1] - h\n",
" if newCSL0 > colSumLines[0]:\n",
" colSumLines[0] = newCSL0\n",
" newCSL5 = colSumLines[4] + h\n",
" if newCSL5 < colSumLines[5]:\n",
" colSumLines[5] = newCSL5\n",
" \n",
" w1 = colSumLines[2] - colSumLines[1]\n",
" w2 = colSumLines[3] - colSumLines[2]\n",
" w3 = colSumLines[4] - colSumLines[3]\n",
" w = max(w1, w2, w3)\n",
" \n",
" newRSL0 = rowSumLines[1] - w\n",
" if newRSL0 > rowSumLines[0]:\n",
" rowSumLines[0] = newRSL0\n",
" newRSL5 = rowSumLines[4] + w\n",
" if newRSL5 < rowSumLines[5]:\n",
" rowSumLines[5] = newRSL5\n",
"\n",
" #print(\"rows2\", rowSumLines)\n",
" #print(\"cols2\", colSumLines)\n",
"\n",
" #just display\n",
" if \"diceRaw\" in generate:\n",
" plt.figure(figsize=(10,10))\n",
" i = 1\n",
" for y in range(5):\n",
" for x in range(5):\n",
" plt.subplot(5,5,i)\n",
" i += 1\n",
" plt.xticks([])\n",
" plt.yticks([])\n",
" plt.grid(False)\n",
" minX = colSumLines[x]\n",
" maxX = colSumLines[x+1]\n",
" minY = rowSumLines[y]\n",
" maxY = rowSumLines[y+1]\n",
" crop_img = warpgray[minY:maxY, minX:maxX]\n",
" plt.imshow(crop_img, cmap=plt.cm.gray)\n",
" if normalPlots:\n",
" plt.show(block=False)\n",
" else:\n",
" resultImages[\"diceRaw\"] = plotToImg()\n",
"\n",
" \n",
" if \"dice\" in generate:\n",
" plt.figure(figsize=(10,10))\n",
" i = 1\n",
"\n",
" letterResize = 30\n",
" #make square, resize, display, and save to an array\n",
" letterImgs = []\n",
" for y in range(5):\n",
" letterImgRow = []\n",
" for x in range(5):\n",
" minX = colSumLines[x]\n",
" maxX = colSumLines[x+1]\n",
" minY = rowSumLines[y]\n",
" maxY = rowSumLines[y+1]\n",
" w = maxX - minX\n",
" h = maxY - minY\n",
" #print(\"w,h 1:\", w, h)\n",
" d = abs(w - h)\n",
" if d > 0:\n",
" if int(d/2) == d/2:\n",
" #even difference\n",
" d1 = d2 = int(d/2)\n",
" else:\n",
" #odd difference\n",
" d1 = int((d-1)/2)\n",
" d2 = int((d+1)/2)\n",
" if w > h:\n",
" #wider than it is tall\n",
" minX += d1\n",
" maxX -= d2\n",
" else:\n",
" #taller than it is wide\n",
" minY += d1\n",
" maxY -= d2\n",
" #print(\"w,h 2:\", maxX-minX, maxY-minY)\n",
" crop_img = warpgray[minY:maxY, minX:maxX]\n",
" letterImg = cv2.resize(crop_img, (letterResize,letterResize), interpolation=cv2.INTER_AREA)\n",
" if \"dice\" in generate:\n",
" plt.subplot(5,5,i)\n",
" i += 1\n",
" plt.xticks([])\n",
" plt.yticks([])\n",
" plt.grid(False)\n",
" plt.imshow(letterImg, cmap=plt.cm.gray)\n",
" letterImgRow.append(letterImg)\n",
" letterImgs.append(letterImgRow)\n",
"\n",
" if \"dice\" in generate:\n",
" if normalPlots:\n",
" plt.show(block=False)\n",
" else:\n",
" resultImages[\"dice\"] = plotToImg()\n",
" \n",
" return resultImages, letterImgs\n",
"\n",
"def findAndShowBoggleBoard(imgDir, imgFilename):\n",
" imgPath = imgDir + \"/\" + imgFilename\n",
"\n",
" #if True:\n",
" try:\n",
" print(imgPath)\n",
" image = cv2.imread(imgPath)\n",
" generate = (\"debugimage\", \"debugmask\", \"contourPlotImg\", \"warpedimage\", \"imgSumPlotImg\", \"diceRaw\", \"dice\")\n",
" #generate = (\"dice\")\n",
" #normalPlots = False\n",
" normalPlots = True\n",
" #harshErrors = False\n",
" harshErrors = True\n",
" imgs, letterImgs = findBoggleBoard(image, normalPlots, harshErrors, generate)\n",
" for title, img in imgs.items():\n",
" #print(\"title, img:\", title, img)\n",
" #cv2.imshow(title, img)\n",
" imshow_fit(title, img)\n",
" if len(imgs) > 0:\n",
" waitForKey()\n",
" else:\n",
" waitForConsoleEnter()\n",
" return letterImgs\n",
" except Exception as e:\n",
" #print(str(e))\n",
" print(traceback.format_exc())\n",
" waitForConsoleEnter()\n",
" return None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#process 1 image\n",
"letters5x5grid = findAndShowBoggleBoard(IMAGE_DIR, IMAGE_FILE)\n",
"print(len(letters5x5grid[0][0]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#https://www.tensorflow.org/tutorials/keras/classification\n",
"\n",
"from __future__ import absolute_import, division, print_function, unicode_literals\n",
"\n",
"# TensorFlow and tf.keras\n",
"import tensorflow as tf\n",
"from tensorflow import keras\n",
"from tensorflow.keras import datasets, layers, models\n",
"\n",
"# Helper libraries\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"import json\n",
"\n",
"#DATA_FILE=\"/home/johanv/downloads/labelled.json\"\n",
"DATA_FILE=\"/home/johanv/nextcloud/projects/boggle2.0/labelled.json\"\n",
"#DATA_FILE=\"/home/johanv/nextcloud/projects/boggle2.0/labelled-20200124_155523.mp4.json\"\n",
"\n",
"#import os\n",
"#DATA_FILE = \"/labelled.json\"\n",
"#DATA_URL = \"https://drive.confuzer.cloud/index.php/s/2fQ5KkGi3Bi2SLq/download\"\n",
"#os.system(\"curl \" + DATA_URL + \" > \" + DATA_FILE)\n",
"\n",
"MODEL_SAVE_FILE=\"/home/johanv/nextcloud/projects/boggle2.0/model.h5\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(tf.__version__)\n",
"\n",
"#fashion_mnist = keras.datasets.fashion_mnist\n",
"#(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n",
"#class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n",
"# 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']\n",
"\n",
"with open(DATA_FILE, 'r') as f:\n",
" data = json.load(f)\n",
"\n",
"IMG_DIM = 30\n",
"\n",
"images_in = data[\"imgs\"]\n",
"labels_in = data[\"labels\"]\n",
"\n",
"images = []\n",
"labels = []\n",
"for rot in range(4):\n",
" for i in range(len(images_in)):\n",
" #https://artemrudenko.wordpress.com/2014/08/28/python-rotate-2d-arraymatrix-90-degrees-one-liner/\n",
" images_in[i] = list(zip(*images_in[i][::-1])) #rotate 90 degrees (still the same letter!)\n",
" images.extend(images_in)\n",
" labels.extend(labels_in)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"images = np.array(images, dtype=np.uint8).reshape((-1, IMG_DIM, IMG_DIM, 1))\n",
"\n",
"split = int(0.15 * len(images))\n",
"\n",
"train_images = np.array(images[split:],dtype=np.uint8)\n",
"train_labels = np.array(labels[split:],dtype=np.uint8)\n",
"\n",
"test_images = np.array(images[:split],dtype=np.uint8)\n",
"test_labels = np.array(labels[:split],dtype=np.uint8)\n",
"\n",
"class_names = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n",
"\n",
"L = len(class_names)\n",
"\n",
"print(train_images.shape)\n",
"print(train_labels.shape)\n",
"print(test_images.shape)\n",
"print(test_labels.shape)\n",
"\n",
"# plt.figure()\n",
"# plt.imshow(train_images[1])\n",
"# plt.colorbar()\n",
"# plt.grid(False)\n",
"# plt.show()\n",
"\n",
"train_images = train_images / 255.0\n",
"test_images = test_images / 255.0\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.figure(figsize=(10,10))\n",
"for i in range(25):\n",
" print(\"{}/25\".format(i))\n",
" plt.subplot(5,5,i+1)\n",
" plt.xticks([])\n",
" plt.yticks([])\n",
" plt.grid(False)\n",
" plt.imshow(train_images[i].reshape((IMG_DIM, IMG_DIM)), cmap=plt.cm.binary)\n",
" plt.xlabel(class_names[train_labels[i]])\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = tf.keras.models.load_model(MODEL_SAVE_FILE)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"predictions = model.predict(test_images)\n",
"print(\"predictions[0]: \", predictions[0])\n",
"print(\"np.argmax(predictions[0]): \", np.argmax(predictions[0]))\n",
"print(\"test_labels[0]: \", test_labels[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def get_label(predictions_array):\n",
" predicted_label = np.argmax(predictions_array)\n",
"# if class_names[predicted_label] == 'E' and 100*np.max(predictions_array) < 99.4:\n",
"# predicted_label = class_names.index('Q')\n",
"# if class_names[predicted_label] == 'T' and 100*np.max(predictions_array) < 96:\n",
"# predicted_label = class_names.index('L')\n",
"# if class_names[predicted_label] == 'U' and 100*np.max(predictions_array) < 67:\n",
"# predicted_label = class_names.index('L')\n",
"# if class_names[predicted_label] == 'R' and 100*np.max(predictions_array) < 85:\n",
"# predicted_label = class_names.index('U')\n",
" return predicted_label\n",
"\n",
"def plot_image(i, predictions_array, true_label, img):\n",
" predictions_array, true_label, img = predictions_array, true_label[i], img[i]\n",
" plt.grid(False)\n",
" plt.xticks([])\n",
" plt.yticks([])\n",
"\n",
" plt.imshow(img.reshape((IMG_DIM, IMG_DIM)), cmap=plt.cm.binary)\n",
"\n",
" predicted_label = get_label(predictions_array)\n",
" if predicted_label == true_label:\n",
" color = 'blue'\n",
" else:\n",
" color = 'red'\n",
"\n",
" plt.xlabel(\"{} {:2.2f}% ({})\".format(class_names[predicted_label],\n",
" 100*np.max(predictions_array),\n",
" class_names[true_label]),\n",
" color=color)\n",
"\n",
"def plot_value_array(i, predictions_array, true_label):\n",
" predictions_array, true_label = predictions_array, true_label[i]\n",
" plt.grid(False)\n",
" plt.xticks(range(L))\n",
" plt.yticks([])\n",
" thisplot = plt.bar(range(L), predictions_array, color=\"#777777\")\n",
" plt.ylim([0, 1])\n",
" predicted_label = get_label(predictions_array)\n",
"\n",
" thisplot[predicted_label].set_color('red')\n",
" thisplot[true_label].set_color('blue')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"i = 0\n",
"plt.figure(figsize=(6,3))\n",
"plt.subplot(1,2,1)\n",
"plot_image(i, predictions[i], test_labels, test_images)\n",
"plt.subplot(1,2,2)\n",
"plot_value_array(i, predictions[i], test_labels)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"i = 12\n",
"plt.figure(figsize=(6,3))\n",
"plt.subplot(1,2,1)\n",
"plot_image(i, predictions[i], test_labels, test_images)\n",
"plt.subplot(1,2,2)\n",
"plot_value_array(i, predictions[i], test_labels)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Plot the first X test images, their predicted labels, and the true labels.\n",
"# Color correct predictions in blue and incorrect predictions in red.\n",
"for j in range(1):\n",
" num_rows = 5\n",
" num_cols = 3\n",
" num_images = num_rows*num_cols\n",
" plt.figure(figsize=(2*2*num_cols, 2*num_rows))\n",
" for i in range(num_images):\n",
" plt.subplot(num_rows, 2*num_cols, 2*i+1)\n",
" plot_image(i+j*num_images, predictions[i+j*num_images], test_labels, test_images)\n",
" plt.subplot(num_rows, 2*num_cols, 2*i+2)\n",
" plot_value_array(i+j*num_images, predictions[i+j*num_images], test_labels)\n",
" plt.tight_layout()\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Grab an image from the test dataset.\n",
"img = test_images[1]\n",
"\n",
"print(img.shape)\n",
"\n",
"# Add the image to a batch where it's the only member.\n",
"img = (np.expand_dims(img,0))\n",
"\n",
"print(img.shape)\n",
"\n",
"predictions_single = model.predict(img)\n",
"\n",
"print(predictions_single)\n",
"\n",
"plot_value_array(1, predictions_single[0], test_labels)\n",
"_ = plt.xticks(range(L), class_names, rotation=45)\n",
"\n",
"print(\"np.argmax(predictions_single[0]): \", np.argmax(predictions_single[0]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#show letters5x5grid\n",
"letters5x5gridFlat = []\n",
"for row in letters5x5grid:\n",
" letters5x5gridFlat.extend(row)\n",
"\n",
" \n",
"letters5x5gridLabelsStr = \"RONLICTSSDNMPPNUAEQIHINRM\"\n",
"letters5x5gridLabels = [class_names.index(letter) for letter in letters5x5gridLabelsStr]\n",
"\n",
"num_rows = 5\n",
"num_cols = 5\n",
"num_images = num_rows*num_cols\n",
"plt.figure(figsize=(2*num_cols, 2*2*num_rows))\n",
"for row in range(num_rows):\n",
" for col in range(num_cols):\n",
" letterImg = letters5x5grid[row][col]\n",
" letterImg = letterImg / 255\n",
" # print(letterImg.shape)\n",
" letterImg = (np.expand_dims(letterImg,0))\n",
" letterImg = (np.expand_dims(letterImg,axis=3))\n",
"\n",
" # print(letterImg.shape)\n",
" pred = model.predict(letterImg)[0]\n",
" # print(np.argmax(pred))\n",
"\n",
" i = col+row*num_cols\n",
" plt.subplot(2*num_rows, num_cols, col+2*row*num_cols+1)\n",
" plot_image(i, pred, letters5x5gridLabels, letters5x5gridFlat)\n",
" plt.subplot(2*num_rows, num_cols, col+2*row*num_cols+num_cols+1)\n",
" plot_value_array(i, pred, letters5x5gridLabels)\n",
"plt.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}