return nonZero;}/* * mdIntraPredLuma4x4: * * Parameters: *      pMb                   Macroblock coding object *      picWidth              Horizontal size of the frame *      picType               Type of the frame (intra/inter) *      bestMbSad             Best MB sad so far *      modeDecision          How mode decision should be performed *      intra4x4Cost          Cost for the intra4x4 prediction *      low_complex_prof3     low complexity prof 3 * * Function: *      Build 4x4 predictions and choose the best mode for each block. * * Returns: *      SAD for the best 4x4 intra prediction modes, if RDO is not enabled, *      otherwise, RDO cost of encoding luma is returned. *      pMb->cbpY is also modified in the function. */int mdIntraPredLuma4x4(macroblock_s   *pMb,                        int            picWidth,                        int            picType,                        int            bestMbSad,                        int            modeDecision,                       costMeasure_s *intra4x4Cost,                       int            low_complex_prof3)  {  u_int8 predBlk[IPR_NUM_MODES1][BLK_SIZE][MBK_SIZE];  int modeAvail1[IPR_NUM_MODES1];  int blk8x8, blkIdx;  int blkCount;  int mode;  int bestMode;  int nonZero, exitFlag;  int searchLimit;  int mostProbableMode;  int mbModeBits;  int *mbAvailMap;  int actualBlkSad=0;  costMeasure_s bestBlkCost, blkCost;  pMb->cbpY = 0;  mbAvailMap = pMb->mbAvailMapIntra;  intra4x4Cost->mse  = 0;  intra4x4Cost->sad  = 0;  pMb->actualIntra4x4Sad = 0;#ifndef DISABLE_RDO  if (modeDecision == MODE_DECISION_RDO)  {    intra4x4Cost->cost = 0;    searchLimit = MAX_COST;  }  else#endif  {    // used for mbType, 5 bits in P-frame, 1 bit in I-frame    intra4x4Cost->cost = pMb->lambda * (IS_SLICE_P(picType) ? 5 : 1);     searchLimit = bestMbSad;  }  mbModeBits = 0;  blkCount = 0;  exitFlag = 0;  if (intra4x4Cost->cost + 16 * pMb->lambda >= searchLimit)  {    exitFlag = 1;    intra4x4Cost->cost += 16 * pMb->lambda;  }  for (blk8x8 = 0; (blk8x8 < 16) && (! exitFlag); blk8x8 += 4)  {    int numNonZeroBlks, nonZeroBlkFlag;    int blk8x8ModeBits, blk8x8CoeffBits;    numNonZeroBlks = 0;    blk8x8CoeffBits = 0;    blk8x8ModeBits = 0;    for (blkIdx = blk8x8; (blkIdx < (blk8x8 + 4)) && (! exitFlag); blkIdx ++)    {      int upMode, leftMode, blkAddr;      u_int8 *origBlk;      u_int8 *recoY;      int blk4x4BestModeBits;      int blk4x4BestNumCoeffs;      int blkX, blkY;      blkX = blkRasterOrder[blkIdx] & 3;      blkY = blkRasterOrder[blkIdx] >> 2;      blkAddr = blkY * 4 + blkX;      /* Get ipr modes of the two of the neighbouring blocks */      leftMode = pMb->blkLeft[blkAddr]->i4x4Mode;      upMode   = pMb->blkUp[blkAddr]->i4x4Mode;      recoY = & pMb->recoY[blkY * BLK_SIZE * picWidth + blkX*BLK_SIZE];      /* Make all possible 4x4 predictions for current block */      iprGetPredLuma4x4(predBlk, modeAvail1, recoY,         picWidth, leftMode, upMode, pMb->i4x4CornersAvail[blkY][blkX]);      // at least one is IPR_MODE_NA      if ((upMode + leftMode) >=  IPR_MODE_NA)        mostProbableMode = IPR_MODE_DC;      else        mostProbableMode = min(upMode, leftMode);      bestMode = 0;      blk4x4BestModeBits = 0;      blk4x4BestNumCoeffs = 0;      origBlk = & pMb->origY[blkY * BLK_SIZE][blkX * BLK_SIZE];      nonZeroBlkFlag = 0;      bestBlkCost = initCostMeasure;      blkCost     = initCostMeasure;      /* Choose the best mode for the current block */            for (mode = 0; mode < IPR_NUM_MODES1; mode++) {                if (modeAvail1[mode]) {          int numCoefs;          blkCost.sad = pMb->lambda * (mode == mostProbableMode ? 1 : 4);          if (modeDecision == MODE_DECISION_HADAMARD)            blkCost.sad += traDiffSATD4x4(            (u_int8 (*)[MBK_SIZE]) origBlk,            (u_int8 (*)[MBK_SIZE]) predBlk[mode]) >> 1;          else            blkCost.sad += traDiffSAD4x4(            (u_int8 (*)[MBK_SIZE]) origBlk,            (u_int8 (*)[MBK_SIZE]) predBlk[mode]);          nonZero  = 1;          numCoefs = 0;          /* Compute total SAD for the current mode by adding the start */          /* 'handicap' and the computed SAD                            */#ifndef DISABLE_RDO          if (modeDecision == MODE_DECISION_RDO)          {                        nonZero = encoRecoBlock((u_int8 (*)[MBK_SIZE]) origBlk, predBlk[mode],               pMb->coefY[blkIdx], recoY, picWidth, picType, pMb->qp);            /* If there were non-zero coefficients, decode block */            // perform entropy encoding            bibInit(& pMb->mbBs);            blkCost.bits = streamSend4x4Blk(& pMb->mbBs, pMb, 1,              COMP_LUMA, blkX, blkY,              pMb->coefY[blkIdx], BLK_CAT_Y, & numCoefs);            blkCost.bits += (mode == mostProbableMode) ? 1 : 4;            blkCost.mse   = CalculateSsd              (origBlk, MBK_SIZE, recoY, picWidth, BLK_SIZE, BLK_SIZE);            // calculate the MSE            blkCost.cost = blkCost.mse + pMb->rdoLambda * blkCost.bits;          }          else#endif          blkCost.cost = blkCost.sad;          if (blkCost.cost < bestBlkCost.cost) {            bestBlkCost   = blkCost;            bestMode      = mode;            blk4x4BestNumCoeffs = numCoefs;            actualBlkSad = blkCost.sad - pMb->lambda * (mode == mostProbableMode ? 1 : 4);             blk4x4BestModeBits  = (mode == mostProbableMode) ? 1 : 4;             nonZeroBlkFlag = nonZero;          }       }        // early termination in for mode        if ( (low_complex_prof3) && (bestBlkCost.cost < TH_IPR_MODE  ) )        {          break;        }      }      // store number of coeffs of the best mode, for future prediction      pMb->current[blkY][blkX].i4x4Mode = (int8) bestMode;      pMb->current[blkY][blkX].numLumaCoefs =         (int8) blk4x4BestNumCoeffs;      // this is the information related i4x4Mode to be stored in stream      pMb->ipTab[blkIdx] = (int8) ((bestMode == mostProbableMode) ?        -1 : ((bestMode < mostProbableMode) ? bestMode : bestMode - 1));      numNonZeroBlks  += nonZeroBlkFlag;      blk8x8CoeffBits += bestBlkCost.bits - blk4x4BestModeBits;  //will be zero for non-RDO      blk8x8ModeBits  += blk4x4BestModeBits;       blkCount ++;      intra4x4Cost->cost += bestBlkCost.cost;      intra4x4Cost->sad  += bestBlkCost.sad;      intra4x4Cost->mse  += bestBlkCost.mse;      pMb->actualIntra4x4Sad+=actualBlkSad;      // a little bit look ahead to have earlier termination      if ((intra4x4Cost->cost + pMb->lambda * (16 - blkCount)) >= searchLimit)      {        // at least this amount        intra4x4Cost->cost += pMb->lambda * (16 - blkCount);        exitFlag = 1;        continue;      }      nonZero = encoRecoBlock((u_int8 (*)[MBK_SIZE]) origBlk, predBlk[bestMode],         pMb->coefY[blkIdx], recoY, picWidth, picType, pMb->qp);      /* If there were non-zero coefficients, decode block */      if (nonZero > 0)        /* Update luma CBP */        pMb->cbpY |= 1 << blkIdx;    }    mbModeBits += blk8x8ModeBits;     if (numNonZeroBlks)      intra4x4Cost->bits += blk8x8CoeffBits;    else      intra4x4Cost->bits += 2;      // do not want to be too biased  }  intra4x4Cost->bits += mbModeBits;  // we re-calculate the sad here#ifndef DISABLE_RDO  if (modeDecision == MODE_DECISION_RDO) {    pMb->intra4x4CbpY = pMb->cbpY;    intra4x4Cost->cost = intra4x4Cost->mse + intra4x4Cost->bits * pMb->rdoLambda;  }  else#endif  {    // sad is used as cost above    intra4x4Cost->sad  = intra4x4Cost->cost;  }  return intra4x4Cost->cost;}/* * * mdIntraPredChroma: * * Parameters: *      pMb                   macroblock information *      orig                  Original pixels *      pred                  Return pointer for predicted pixels * * Function: *      Build 8x8 chroma predictions and choose the best mode for the *      macroblock. * * Returns: *      SAD of the best chroma intra mode * */int mdIntraPredChroma(macroblock_s *pMb,                       int chromaModeAvail[IPR_CHROMA_NUM_MODES],                      int hadamard){  int bestMode;  int bestCost;  int mode;  int currentCost;  int c;  int i, j;  int dcCoeffs[2][2];  /*   * Build 8x8 intra prediction   */  bestMode = -1;  bestCost  = INT_MAX;  for (mode = 0; mode < IPR_CHROMA_NUM_MODES; mode++)   {    if (chromaModeAvail[mode])     {      currentCost = 0;      for (c = 0; c < MBK_SIZE; c += MBK_SIZE/2) {        for (j = 0; j < BLK_PER_MB/2; j ++) {          for (i = 0; i < BLK_PER_MB/2; i ++)          {            if (hadamard)              currentCost += traDiffSATD4x4RetDc(              (u_int8 (*)[MBK_SIZE]) &pMb->origC[j*BLK_SIZE][c + i*BLK_SIZE],              (u_int8 (*)[MBK_SIZE]) &pMb->predIntraC[mode][j*BLK_SIZE][c + i*BLK_SIZE],              & dcCoeffs[j][i]);            else              currentCost += traDiffSAD4x4(              (u_int8 (*)[MBK_SIZE]) &pMb->origC[j*BLK_SIZE][c + i*BLK_SIZE],              (u_int8 (*)[MBK_SIZE]) &pMb->predIntraC[mode][j*BLK_SIZE][c + i*BLK_SIZE]);          }        }        if (hadamard) {          traDCT2x2(dcCoeffs);          // DC (>> 2), 2x2 Hadamard (<< 1) to match 4x4 Hadamard, end up (>> 1)          currentCost += (abs(dcCoeffs[0][0]) + abs(dcCoeffs[0][1]) +            abs(dcCoeffs[1][0]) + abs(dcCoeffs[1][1])) >> 1;        }      }      if (hadamard)        currentCost = currentCost >> 1;      currentCost = currentCost + uvlcBitsUnsigned[mode] * pMb->lambda;      if (currentCost < bestCost ) {        bestCost = currentCost;        bestMode = mode;      }    }  }  pMb->intraModeChroma = bestMode;  return bestCost;}extern int8 mcpModePartWidth[5];extern int8 mcpModePartHeight[5];static void getChannelDistortion(macroblock_s *pMb,                               refFrmBuf_s  **refBufList,                               int          ssd0,                               int          *distortion){  int blkIdxX, blkIdxY;  int shapeX, shapeY, mbPartW, mbPartH;  int i0, j0;  int i1, j1;  int i2, j2;  int k0, l0;  int picWidth;  int picHeight;  int mbIdxRef;  int mbkPerLine;  int mbAddr;  int *pDistortion;  distortion[0] = 0;  picWidth = refBufList[0]->picWidth;  picHeight = refBufList[0]->picHeight;  mbkPerLine = picWidth / MBK_SIZE;  mbAddr = pMb->idxY * picWidth / MBK_SIZE + pMb->idxX ;  shapeX = mcpModePartWidth[pMb->interMode];  shapeY = mcpModePartHeight[pMb->interMode];  mbPartW = shapeX * BLK_SIZE;  mbPartH = shapeY * BLK_SIZE;  // 1:    (1-p) * Dc(n-1, j)  for (blkIdxY = 0; blkIdxY < BLK_PER_MB; blkIdxY += shapeY) {    for (blkIdxX = 0; blkIdxX < BLK_PER_MB; blkIdxX += shapeX) {      blkState_s *pBlkState;      // the starting position of current block in pixel: k0, l0      k0 = (pMb->idxX * BLK_PER_MB + blkIdxX) * BLK_SIZE;      l0 = (pMb->idxY * BLK_PER_MB + blkIdxY) * BLK_SIZE;      // Absolute motion vector coordinates of the macroblock      pBlkState = & pMb->current[blkIdxY][blkIdxX];      pDistortion = refBufList[pBlkState->ref]->channelDistortion;      i0 = k0 * 4 + pBlkState->mv.x;      j0 = l0 * 4 + pBlkState->mv.y;      // the starting position of the ref block in pixel: i0, j0      i0 = i0 / 4;      j0 = j0 / 4;      if (i0 < 0) i0 = 0;      if (j0 < 0) j0 = 0;      if (i0 >= picWidth) i0 = picWidth - 1;      if (j0 >= picHeight) j0 = picHeight - 1;      // calculate the distortion here:      for (j1 = j0; j1 < j0 + mbPartH; j1++) {        for (i1 = i0; i1 < i0 + mbPartW; i1++) {          i2 = i1;          j2 = j1;          if (i2 >= picWidth) i2 = picWidth - 1;          if (j2 >= picHeight) j2 = picHeight - 1;          mbIdxRef  = (j2 / MBK_SIZE) * mbkPerLine + i2 / MBK_SIZE;          distortion[0] += pDistortion[mbIdxRef];  //  / 256.0        }      }    }  }  distortion[0] = distortion[0] >> 8; //  / 256  distortion[0] = distortion[0] * (100-pMb->plr);  // 2:    p * Dc(n-1, i)  distortion[0] += refBufList[0]->channelDistortion[mbAddr] * pMb->plr;  // 3:    p * ssd  distortion[0] += ssd0 * pMb->plr;  distortion[0] = distortion[0] / 100;}static void mdInterModeCost(macroblock_s   *pMb,