php - Update about 1 million rows in MySQL table every 1 hour

Solution:

Trying limitting the update instead of updating all at once. Update in chunks.

function cron_rows_update() {
   $num_of_members = $this->members_model->get_num_of_members();
   $limit_per_run = 150;
   $total_run = (int)$num_of_members/$per_run;
   
   for( $i = 0; $i <= $total_run; $i++ ) {
      $offset = $i * $limit_per_run;
      $this->members_model->rows_update($offset, $limit_per_run);
   }

}

and in rows_update()

function rows_update($offset, $limit_per_run) {
    
    **

    $this->db->limit($offset, $limit_per_run);
    $query = $this->db->get();
    
    $arrUpdateBatchData = [];
    
    while ($row = $query->unbuffered_row('array'))
    {
        // calculations and create array for batch update
    }

    // update in batch limiting to selected number of records
    if (count($arrUpdateBatchData) > 0)
    {
        $this->db->update_batch('members', $arrUpdateBatchData, 'UserID');
    }
}

You can also try using a timer function in controller to trigger update after a certain interval of time and increase the server capacities as mentioned in comments.

Read more here for further notes on high number of record updates as fast as possile and to decrease the load on server.

Solution:

Apart from the fact that your code is a little bit confusing ($PER_part2 and $PER_part3 are unused) i would do the following:

The script will not work with result_array for more than 1 million iterations. The reason is, result_array stores all data in an array - and sooner or later you'll get a memory limit problem.

In order to avoid this you have to use unbuffered_row. This method returns a single result row without prefetching the whole result in memory.

Take a look at their documentation here. (section unbuffered_row)

The next thing i would change are your if blocks - i would use the if/else syntax here. (it isn't that big of a difference but considering your amount of rows - it might helps)

Also, i outsourced the block where you calculate your rating twice with the same logic behind.

Basically the following should work:

function rows_update() 
{
    $currency_numbers_after_dot = $this->currencies_model->get_currency('ONE', 'My currencty')['numbers_after_dot'];
    $game_currency_percentage_max = $this->settings_model->get_settings_details('game_rating_percentage_max')['value'];
    $game_currency_speed_in_hour = $this->settings_model->get_settings_details('game_currency_speed_in_hour')['value'];
    $this->db->from('members');
    $query = $this->db->get();
    
    $arrUpdateBatchData = [];
    
    while ($row = $query->unbuffered_row('array'))
    {
        $game_total_balance = round($row['game_vault_balance'] + $row['game_available_balance'], $currency_numbers_after_dot);
        if ($game_total_balance > 0) {
            // Game Rating Calculate
            // Rating Part1
            // Rating Part1_1
            $rating_part1_1 = $this->getRatingPart($row['game_vault_balance']);
            $rating_part1_2 = $this->getRatingPart($game_total_balance);
            // Rating part1_3
            $rating_part1_3 = 0;
            // Rating part1_4
            $rating_part1_4 = 0;
            // Rating part2
            $PER_part2 = '0';
            // Rating part3
            $PER_part3 = '0';
            // Calculate all rating
            $rating = round($rating_part1_1 + $rating_part1_2 + $rating_part1_3 + $rating_part1_4 + $rating_part2 + $rating_part3, 2);
            if ($rating <= 1) {
                $rating_member = $rating;
            }
            elseif ($rating > 1) {
                $rating_member = floor($rating);
            }
            
            // Game balance calculate
            $amount_from_game_vault_in_hour = $game_currency_speed_in_hour / '100' * $row['game_vault_balance'];
            $new_balance_in_hour = ($game_currency_percentage_max / '100') * $row['rating'] * $amount_from_game_vault_in_hour;
            $game_balance_in_hour_amount = $amount_from_game_vault_in_hour + $new_balance_in_hour;
            
            $arrUpdateData = [
                'UserID' => $row['UserID'], 
                'game_vault_balance' => ($row['game_vault_balance'] - $amount_from_game_vault_in_hour),
                'game_available_balance' => ($row['game_available_balance'] - $game_balance_in_hour_amount),
                'rating' => $rating_member,
                'game_rating_and_balance_update_last_time' => date('Y-m-d H:i:s')
            ];
            
            $arrUpdateBatchData[] = $arrUpdateData;
            
        }
        
        if (count($arrUpdateBatchData) > 500)
        {
            $this->db->update_batch('members', $arrUpdateBatchData, 'UserID');
            $arrUpdateBatchData = [];
        }       
    }
    
    //update last items
    if (count($arrUpdateBatchData) > 0)
    {
        $this->db->update_batch('members', $arrUpdateBatchData, 'UserID');
        $arrUpdateBatchData = [];
    }
    return;    
}

function getRatingPart($val)
{
    if ($val == 0) {
        $rating_part = 0; 
    }
    elseif ($val > 0 AND $val < 20) 
    {
        $rating_part = '0.1'; 
    }
    elseif ($val > 20 AND $val < 2000) 
    {
    
        $max_game_vault_balance = 2000;
        $percent = floor($val * 100 / $max_game_vault_balance);
        $additional_rating = 0.05 * $percent / 100;
           
        $rating_part = round(0.1 + $additional_rating, 2); 

    }
    else {
        $rating_part = 0.15; 
    }
    
    return $rating_part;
}