BoggleCV/BoggleCVPipeline.py

#!/usr/bin/python3
import cv2, math, os, json, traceback, io, time
print(cv2.__version__)

import numpy as np
import scipy.signal


#https://www.tensorflow.org/tutorials/keras/classification
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import datasets, layers, models
print(tf.__version__)

MODEL_FILE="/srv/boggle/model.h5"
IMG_DIM = 30
class_names = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
L = len(class_names)
model = tf.keras.models.load_model(MODEL_FILE)



RED = (0, 0, 255)
BLUE = (255, 0, 0)
GREEN = (0, 255, 0)
YELLOW = (0, 255, 255)
CONTOUR_THICKNESS = 2
MAX_DISP_DIM = 500

#a generic error
class BoggleError(Exception):
    def __init__(self, arg):
        self.strerror = arg
        self.args = {arg}

def four_point_transform(image, pts, size):
    w = size
    h = size
    ntl = [0, 0]
    ntr = [w, 0]
    nbr = [w, h]
    nbl = [0, h]
    location = np.float32(pts)
    x0 = pts[0][0]
    x2 = pts[2][0]
    #make sure the board ends up rotated the correct way
    if x0 < x2:
        newLocation = np.float32([ntl, nbl, nbr, ntr])
    else:
        newLocation = np.float32([ntr, ntl, nbl, nbr])
    M = cv2.getPerspectiveTransform(location, newLocation)
    warpedimage = cv2.warpPerspective(image, M, (w, h))
    return warpedimage

def resizeWithAspectRatio(image, maxDispDim, inter=cv2.INTER_AREA):
    w = image.shape[0] #TODO width and height are swapped here
    h = image.shape[1]

    ar = w / h
    #print(ar)
    if w > h:
        nw = maxDispDim
        nh = int(maxDispDim / ar)
    elif h > w:
        nh = maxDispDim
        nw = int(maxDispDim / ar)
    else:
        nh = maxDispDim
        nw = maxDispDim

    dim = None
    (h, w) = image.shape[:2]

    r = nw / float(w)
    dim = (nw, int(h * r))

    return cv2.resize(image, dim, interpolation=inter)

def imshow_fit(name, img, maxDispDim=MAX_DISP_DIM):
    if maxDispDim is not None:
        img = resizeWithAspectRatio(img, maxDispDim)
    cv2.imshow(name, img)

def findBestCtr(contours):
    bestArea = 0
    bestCtr = None
    for i, ctr in enumerate(contours):
        area = cv2.contourArea(ctr)
        # perimeter = cv2.arcLength(ctr, True)
        if area > bestArea:
            bestArea = area
            bestCtr = ctr
    return bestCtr


def angleEveryFew(ctr, step):
    angles = []
    # dists = []
    prev = ctr[-1]
    for i in range(0, len(ctr), step):
        curr = ctr[i]
        px, py = prev[0]
        cx, cy = curr[0]
        # angle = (px-cx)/(py-cy)
        angle = math.atan2(cy - py, cx - px)
        angles.append(angle)
        # dist = math.hypot(cx-px, cy-py)
        # dists.append(dist)
        prev = curr
    return angles


def angleAvg(angles):
    x = 0
    y = 0
    for angle in angles:
        x += math.cos(angle)
        y += math.sin(angle)
    return math.atan2(y, x)


def angleDiffAbs(angle1, angle2):
    return abs(((angle1 - angle2 + math.pi) % (2 * math.pi)) - math.pi)


def runningAvg(angles, history):
    result = []
    for i in range(len(angles)):
        # avg = 0
        vals2 = []
        for j in range(history):
            val = angles[int(i + j - history / 2) % len(angles)]
            vals2.append(val)
            # avg += val
        # avg /= history
        avg = angleAvg(vals2)
        result.append(avg)
    return result


def diffAbs(vals):
    diffs = []
    prev = vals[-1]
    for i in range(0, len(vals), 1):
        curr = vals[i]
        diff = angleDiffAbs(prev, curr) * 10
        diffs.append(diff)
        prev = curr
    return diffs


def debounce(bools, history):
    result = []
    for i in range(len(bools)):
        total = 0
        for j in range(history):
            val = bools[int(i + j - history / 2) % len(bools)]
            total += val
        result.append(int(total >= history / 2))
    return result


def findGaps(diffs):
    wasGap = True
    seamI = 0
    for i, diff in enumerate(diffs):
        isGap = diff
        if not isGap and not wasGap:
            seamI = i
            break
        wasGap = isGap

    xs = range(len(diffs))
    xs = [x for x in xs]
    xs_a = xs[seamI:]
    xs_a.extend(xs[:seamI])
    diffs_a = diffs[seamI:]
    diffs_a.extend(diffs[:seamI])

    xs2 = []
    diffs2 = []
    gapsStart = []
    gapsStartY = []
    gapsEnd = []
    gapsEndY = []
    wasGap = None
    gapwidth = 0
    for x, diff in zip(xs_a, diffs_a):
        isGap = diff
        if wasGap is None:
            wasGap = isGap
        if isGap:
            xs2.append(x)
            diffs2.append(diff)
            gapwidth += 1
        if isGap and not wasGap:
            gapsStart.append(x)
            gapsStartY.append(.5)
        if not isGap and wasGap:
            gapsEnd.append(x)
            gapsEndY.append(gapwidth)
            gapwidth = 0
        wasGap = isGap
    return gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2
    # gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]
    # return gaps, diffs, xs2


def top4gaps(gaps):
    def length_sort(gap):
        return -gap[2]

    gaps2 = sorted(gaps, key=length_sort)
    gaps2 = gaps2[:4]
    return [i for i in gaps if i in gaps2]


def invertGaps(gaps):
    segments = []
    prev = gaps[-1]
    for curr in gaps:
        segments.append((prev[1], curr[0]))
        prev = curr
    return segments


def findSidePoints(segments, ctr, step):
    sidePoints = []
    for seg in segments:
        if seg[1] > seg[0]:
            sidePoints.append(ctr[seg[0] * step:seg[1] * step])
        else:
            sidePoints.append(ctr[seg[0] * step:, :seg[1] * step])
    return sidePoints


def getEndVals(arr, fraction):
    if fraction >= 0.5: return arr
    l = len(arr)
    keep = int(fraction * l)
    keep = max(keep, 1)
    return np.concatenate((arr[:keep], arr[l - keep:]))


def fitSidePointsToLines(sidePoints):
    lines = []
    for sp in sidePoints:
        xs = np.zeros(len(sp), int)
        ys = np.zeros(len(sp), int)
        for i, pt in enumerate(sp):
            x, y = pt[0] #TODO if pt is empty
            xs[i] = x
            ys[i] = y
        lines.append(np.polyfit(xs, ys, 1))
    return lines


def findCorners(lines):
    points = []

    prev = lines[-1]
    for curr in lines:
        a1, b1 = prev
        a2, b2 = curr

        x = (b2 - b1) / (a1 - a2)
        y = np.polyval(curr, x)
        #if math.isnan(x): x = 0 #TODO
        #if math.isnan(y): y = 0
        points.append((int(x), int(y)))
        prev = curr
    return points


def drawLinesAndPoints(image, lines, points):
    width = image.shape[1]

    for l in lines:
        y1 = int(np.polyval(l, 0))
        y2 = int(np.polyval(l, width - 1))
        cv2.line(image, (0, y1), (width - 1, y2), RED, CONTOUR_THICKNESS)

    for point in points:
        cv2.circle(image, point, 4, BLUE, 3)

#https://scipy-cookbook.readthedocs.io/items/SignalSmooth.html
#window: np.ones (flat), np.hanning, np.hamming, np.bartlett, np.blackman
def smooth(x, window_len=11, window=np.hanning):
    if x.ndim != 1:
        raise ValueError("smooth only accepts 1 dimension arrays.")

    if x.size < window_len:
        raise ValueError("Input vector needs to be bigger than window size.")

    if window_len<3:
        return x

    s=np.r_[x[window_len-1:0:-1], x, x[-2:-window_len-1:-1]]
    w = window(window_len)
    y = np.convolve(w / w.sum(), s, mode='valid')
    return y


def findRowsOrCols(img, doCols, smoothFactor, ax):
    smoothFactor = int(smoothFactor * img.shape[0])
    #print("smoothFactor", smoothFactor)
    
    if doCols:
        title = "colSum"
        imgSum = cv2.reduce(img, 0, cv2.REDUCE_AVG, dtype=cv2.CV_32S)
        imgSum = imgSum[0]
    else:
        #row sum
        title = "rowSum"
        imgSum = cv2.reduce(img, 1, cv2.REDUCE_AVG, dtype=cv2.CV_32S)
        imgSum = imgSum.reshape(len(imgSum))
    
    imgSumSmooth = smooth(imgSum, smoothFactor*2)

    #https://qingkaikong.blogspot.com/2018/07/find-peaks-in-data.html
    #peaks_positive, _ = scipy.signal.find_peaks(imgSumSmooth, height=200, threshold = None, distance=60)
    dips, props = scipy.signal.find_peaks(-imgSumSmooth, height=(None,None), distance=30, prominence=(None,None))
    
    #threshold=(None,None), 
    #, plateau_size=(None,None)
    
    #print(props)

    prs = props["prominences"]
    if len(prs) < 6:
        top_6_dips = dips
        #print ("!!!! less than 6 dips")
        #raise BoggleError("less than 6 dips")
    else:
        prsIdx = sorted(range(len(prs)), key=lambda i: prs[i], reverse=True)
        #print(prsIdx)
        prsIdx = prsIdx[:6]
        #print(prsIdx)
        top_6_dips = [p for i,p in enumerate(dips) if i in prsIdx]

    #fig = plt.figure()
    #ax1 = fig.add_subplot(111)
    
    if ax is not None:
        q = [i for i in range(len(imgSumSmooth))]
        
        ax.plot(q, imgSumSmooth, 'b-', linewidth=2, label="smooth")
        ax.plot(np.linspace(smoothFactor,len(imgSumSmooth)-smoothFactor, len(imgSum)), imgSum, 'r-', linewidth=1, label=title)

        #ax.plot(
            #[q[p] for p in peaks_positive],
            #[imgSumSmooth[p] for p in peaks_positive],
            #'ro', label = 'positive peaks')
        
        ax.plot(
            [q[p] for p in dips],
            [imgSumSmooth[p] for p in dips],
            'go', label='dips')
        
        ax.plot(
            [q[p] for p in top_6_dips],
            [imgSumSmooth[p] for p in top_6_dips],
            'c.', label='top 6 dips')
        
        ax.legend(loc='best')
        
    #return top_6_dips
    #print("before clip", top_6_dips)
    top_6_dips_scaled = [np.clip(0, p-smoothFactor, len(imgSum)-1) for p in top_6_dips]
    return top_6_dips_scaled


def findBoggleBoard(image, normalPlots=True, harshErrors=False, generate=("debugimage", "debugmask", "contourPlotImg", "warpedimage", "imgSumPlotImg", "diceRaw", "dice")):
    resultImages = {}

    # maskThresholdMin = (108, 28, 12)
    # maskThresholdMax = (125, 255, 241)
    # maskThresholdMin = (108, 28, 6)
    # maskThresholdMax = (130, 255, 241)
    maskThresholdMin = (108, 28, 6)
    maskThresholdMax = (144, 255, 241)
    size = max(image.shape)
    #print("size", size)
    blurAmount = int(.02 * size)
    blurAmount = (blurAmount, blurAmount)
    # blurThreshold = 80
    blurThreshold = 40
    contourApprox = cv2.CHAIN_APPROX_NONE
    # contourApprox = cv2.CHAIN_APPROX_SIMPLE
    # contourApprox = cv2.CHAIN_APPROX_TC89_L1
    # contourApprox = cv2.CHAIN_APPROX_TC89_KCOS

    hsvimg = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    mask = cv2.inRange(hsvimg, maskThresholdMin, maskThresholdMax)

    maskblur = cv2.blur(mask, blurAmount)
    # maskblur = cv2.threshold(maskblur, 80, 255, cv2.THRESH_BINARY_INV)
    maskblur = cv2.inRange(maskblur, (blurThreshold,), (255,))
    
    #API CHANGE: findContours no longer returns a modified image
    #contourimg, contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)
    contours, hierarchy = cv2.findContours(maskblur, cv2.RETR_LIST, contourApprox)
    # print(hierarchy) #TODO
    
    bestCtr = findBestCtr(contours)
    # bestCtr = cv2.convexHull(bestCtr)

    #draw the contours for debugging
    if "debugimage" in generate:
        debugimage = image.copy()
        cv2.drawContours(debugimage, contours, -1, RED, CONTOUR_THICKNESS)
        cv2.drawContours(debugimage, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)
    if "debugmask" in generate:
        debugmask = cv2.cvtColor(maskblur, cv2.COLOR_GRAY2BGR)
        cv2.drawContours(debugmask, contours, -1, RED, CONTOUR_THICKNESS)
        cv2.drawContours(debugmask, [bestCtr], -1, BLUE, CONTOUR_THICKNESS)
    

    step = 10
    avgWindow = 0.1
    debounceFactor = .05
    
    angles = angleEveryFew(bestCtr, step)

    xs = range(len(angles))

    anglesAvg = runningAvg(angles, int(avgWindow * len(angles)))
    diffs = diffAbs(anglesAvg)
    
    avgDiff = np.mean(diffs)
    #print(avgDiff)
    
    binDiffs = [int(i > avgDiff) for i in diffs]
    binDiffs = debounce(binDiffs, int(len(binDiffs) * debounceFactor))
    gapsStart, gapsStartY, gapsEnd, gapsEndY, diffs2, xs2 = findGaps(binDiffs)
    gaps = [i for i in zip(gapsStart, gapsEnd, gapsEndY)]
    
    #scale for viewing on the plot
    gapsEndY2 = gapsEndY
    if len(gapsEndY) > 0:
        q = max(gapsEndY)
        if q > 6: gapsEndY2 = [i * 6 / q for i in gapsEndY]

    if "contourPlotImg" in generate:
        contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, gapsEndY2, normalPlots)
        if contourPlotImg is not None:
            resultImages["contourPlotImg"] = contourPlotImg
    
    if len(gaps) < 4:
        print("!!!! less than 4 gaps")
        if harshErrors:
            raise BoggleError("less than 4 gaps")
        if "contourPlotImg" in generate:
            contourPlotImg = contourPlot(xs, xs2, angles, anglesAvg, diffs, diffs2, gapsStart, gapsStartY, gapsEnd, None, normalPlots)
            if contourPlotImg is not None:
                resultImages["contourPlotImg"] = contourPlotImg
        
        if "debugmask" in generate:
            resultImages["debugmask"] = debugmask
        if "debugimage" in generate:
            resultImages["debugimage"] = debugimage
        return resultImages, None
    
    endFraction = 0.01
    
    gaps = top4gaps(gaps)
    segments = invertGaps(gaps)
    sidePoints = findSidePoints(segments, bestCtr, step)
    sidePoints = [getEndVals(sp, endFraction) for sp in sidePoints]
    #print("sidepoints len", len(sidePoints[0]))
    
    
    lines = fitSidePointsToLines(sidePoints)
    points = findCorners(lines)
    if "debugimage" in generate:
        cv2.drawContours(debugimage, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)
        drawLinesAndPoints(debugimage, lines, points)
        resultImages["debugimage"] = debugimage
    if "debugmask" in generate:
        cv2.drawContours(debugmask, sidePoints, -1, YELLOW, CONTOUR_THICKNESS)
        drawLinesAndPoints(debugmask, lines, points)
        resultImages["debugmask"] = debugmask

    npPoints = np.array(points)
    size = 300
    warpedimage = four_point_transform(image, npPoints, size)
    warpgray = cv2.cvtColor(warpedimage, cv2.COLOR_BGR2GRAY)
    
    if "warpedimage" in generate:
        resultImages["warpedimage"] = warpedimage
    
    if "warpgray" in generate:
        resultImages["warpgray"] = warpgray
    
    smoothFactor = .05
    
    if "imgSumPlotImg" in generate:
        fig, (ax0, ax1) = plt.subplots(2, figsize=(8,10))
    else:
        ax0 = ax1 = None
    
    rowSumLines = findRowsOrCols(warpgray, False, smoothFactor, ax0)
    #print("rows", rowSumLines)
    colSumLines = findRowsOrCols(warpgray, True, smoothFactor, ax1)
    #print("cols", colSumLines)
    
    
    if "imgSumPlotImg" in generate:
        if normalPlots:
            plt.show(block=False)
        else:
            resultImages["imgSumPlotImg"] = plotToImg()


    if len(rowSumLines) < 6 or len(colSumLines) < 6:
        print("!!!! not enough grid lines")
        if harshErrors:
            raise BoggleError("not enough gridlines")
        return resultImages, None
    
    #fix the outermost lines of the board
    h1 = rowSumLines[2] - rowSumLines[1]
    h2 = rowSumLines[3] - rowSumLines[2]
    h3 = rowSumLines[4] - rowSumLines[3]
    h = max(h1, h2, h3)
    
    newCSL0 = colSumLines[1] - h
    if newCSL0 > colSumLines[0]:
        colSumLines[0] = newCSL0
    newCSL5 = colSumLines[4] + h
    if newCSL5 < colSumLines[5]:
        colSumLines[5] = newCSL5
    
    w1 = colSumLines[2] - colSumLines[1]
    w2 = colSumLines[3] - colSumLines[2]
    w3 = colSumLines[4] - colSumLines[3]
    w = max(w1, w2, w3)
    
    newRSL0 = rowSumLines[1] - w
    if newRSL0 > rowSumLines[0]:
        rowSumLines[0] = newRSL0
    newRSL5 = rowSumLines[4] + w
    if newRSL5 < rowSumLines[5]:
        rowSumLines[5] = newRSL5

    #print("rows2", rowSumLines)
    #print("cols2", colSumLines)

    #just display
    if "diceRaw" in generate:
        plt.figure(figsize=(10,10))
        i = 1
        for y in range(5):
            for x in range(5):
                plt.subplot(5,5,i)
                i += 1
                plt.xticks([])
                plt.yticks([])
                plt.grid(False)
                minX = colSumLines[x]
                maxX = colSumLines[x+1]
                minY = rowSumLines[y]
                maxY = rowSumLines[y+1]
                crop_img = warpgray[minY:maxY, minX:maxX]
                plt.imshow(crop_img, cmap=plt.cm.gray)
        if normalPlots:
            plt.show(block=False)
        else:
            resultImages["diceRaw"] = plotToImg()

    
    if "dice" in generate:
        plt.figure(figsize=(10,10))
        i = 1

    letterResize = 30
    #make square, resize, display, and save to an array
    letterImgs = []
    for y in range(5):
        letterImgRow = []
        for x in range(5):
            minX = colSumLines[x]
            maxX = colSumLines[x+1]
            minY = rowSumLines[y]
            maxY = rowSumLines[y+1]
            w = maxX - minX
            h = maxY - minY
            #print("w,h 1:", w, h)
            d = abs(w - h)
            if d > 0:
                if int(d/2) == d/2:
                    #even difference
                    d1 = d2 = int(d/2)
                else:
                    #odd difference
                    d1 = int((d-1)/2)
                    d2 = int((d+1)/2)
                if w > h:
                    #wider than it is tall
                    minX += d1
                    maxX -= d2
                else:
                    #taller than it is wide
                    minY += d1
                    maxY -= d2
            #print("w,h 2:", maxX-minX, maxY-minY)
            crop_img = warpgray[minY:maxY, minX:maxX]
            letterImg = cv2.resize(crop_img, (letterResize,letterResize), interpolation=cv2.INTER_AREA)
            if "dice" in generate:
                plt.subplot(5,5,i)
                i += 1
                plt.xticks([])
                plt.yticks([])
                plt.grid(False)
                plt.imshow(letterImg, cmap=plt.cm.gray)
            letterImgRow.append(letterImg)
        letterImgs.append(letterImgRow)

    if "dice" in generate:
        if normalPlots:
            plt.show(block=False)
        else:
            resultImages["dice"] = plotToImg()
    
    return resultImages, letterImgs


def processImage(filepath):
    image = cv2.imread(filepath)
    _, letters5x5grid = findBoggleBoard(image, normalPlots=False, harshErrors=True, generate=())
    
    lettersGuessed = ""
    confidence = []
    num_rows = 5
    num_cols = 5
    num_images = num_rows*num_cols
    for row in range(num_rows):
        for col in range(num_cols):
            letterImg = letters5x5grid[row][col]
            letterImg = letterImg / 255
#             print(letterImg.shape)
            letterImg = (np.expand_dims(letterImg,0))
            letterImg = (np.expand_dims(letterImg,axis=3))
#             print(letterImg.shape)
            pred = model.predict(letterImg)[0]
            letter = class_names[np.argmax(pred)]
            lettersGuessed += letter
            confidence.append(np.max(pred))
    return lettersGuessed, confidence

"""
# letters5x5gridLabelsStr = "IZEIMFLTYTOSEINHETNORRISU" #00162
# letters5x5gridLabelsStr = "DONLIEEIIESAPYWTAAKLTINRE" #00164
# letters5x5gridLabelsStr = "RAOCODSEERGAPEWORXRHELWNT" #00165
# letters5x5gridLabelsStr = "RONLICTSSDNMPPNUAEQIHINRM" #00170
letters5x5gridLabelsStr = "VANUIRAUHESKEITTDPRCGOUCA" #00171, 00160
# letters5x5gridLabelsStr = "IXESMFLEYTOSEENOETNRRRIWM" #00172, 00161

#process 1 image
IMAGE_FILE = '/home/johanv/nextcloud/projects/boggle2.0/cascademan/categories/5x5/images/00160.jpg'
lettersGuessed, confidence = processImage(IMAGE_FILE)

right = "".join([str(int(a == b)) for a,b in zip(lettersGuessed, letters5x5gridLabelsStr)])

confidence_right = []
confidence_wrong = []

for i, c in enumerate(confidence):
    if right[i] == "1":
        confidence_right.append(c)
    else:
        confidence_wrong.append(c)

print("guess: " + lettersGuessed)
print("real:  " + letters5x5gridLabelsStr)
print("right: " + right)
# print("confidence:", confidence)
# print("confidence_right:", confidence_right)
# print("confidence_wrong:", confidence_wrong)
"""