Processing Takes Ridiculous time when OpenCV detects a positive image

For some reason while I run OpenCV, when it detects a positive image instantly the detector processing time goes up ridiculously. Below in the code I labeled where the time is printed. Normally that output is about a tenth of a second but one it detects a positive image it instantly goes to 50 seconds and more! What causes this?

Note: This project is being done on a raspberry pi zero.

import picamera
from picamera.array import PiRGBArray
import cv2
import time
import numpy as np
import logging

image_path = '/home/pi/photo.bgr'


def DetectWeeds(img):
    time1 = time.time()
    weedDetector = cv2.CascadeClassifier('/home/pi/WeedClassifier/data/cascade.xml')

    found = weedDetector.detectMultiScale(img, minSize=(20, 20))
    time2 = time.time()
    print(time2 - time1)  #PROCESSING TIMER HERE

    amount_found = len(found)
    i = 0
    time.sleep(.1)
#Taking a lot of time when a weed is detected
    if amount_found != 0:
        # There may be more than one
        # sign in the image
        firstset = found[0]
        x, y, width, height = firstset

        imagewidth = 640

        currentx = (x + width) / 2 + x
        currenty = (y + height) / 2 + y

        targetx = imagewidth / 2

        return currentx, targetx, currenty

    if amount_found == 0:
        return 0, -1, 0

def main():
    camera = picamera.PiCamera()
    camera.resolution = (640, 480)
    camera.framerate = 16
    rawCapture = PiRGBArray(camera, size=(640, 480))
    for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):

        image = frame.array

        currentx, targetx, currenty = DetectWeeds(image)

        if currentx != 0:

            if currentx > targetx:

                print("Move Left!")

            if currentx < targetx:
                print("Move Right!")

            if currentx == targetx:
                print("Centered!")
                print("Move Forward Now!")

        if currentx == 0 and targetx == -1:
            print("No Weeds Here")

        rawCapture.truncate(0)
        time.sleep(.08)
main()

from Processing Takes Ridiculous time when OpenCV detects a positive image