I am trying to loop over a large number of files (360,000 files) to convert them from .csv to .dta files (the idea is then to append the data files later).

When I try to do all the files in one loop, I get an error (too many file names). So I did the following outer loop to convert them in groups of 10000 files.

The problem is after 10000 files are converted and saved, I get the error message "observation numbers out of range"

Here is my code:

Code:
clear all

global maindir "C:/Users/soha_/Desktop/MorningStar/1-Inputs"
set trace on
*------------------------------------------------------------------------------*
// File list
filelist , dir("$maindir/Historical Assets/Assets_csv") pattern(*.csv) norecur

// save a copy so that we can load each observation in the loop below
tempfile files
save "`files'"

local rangeinitial=1
forvalues r = 0(5000)360000 { //366715


local range1 = `rangeinitial' + `r'
local range2 = `range1' + 4999

cd "$maindir/Historical Assets"

forvalues i=`range1'/`range2' {

use "`files'" in `i', clear
local f = dirname + "/" + filename
local filename = filename

insheet using "`f'", clear
local name: subinstr local filename ".csv" ""

save "Assets_dta/`name'.dta", replace
}
}
when I set trace on, I get the following:


Code:
- local range1 = `rangeinitial' + `r'
= local range1 = 1 + 10000
- local range2 = `range1' + 4999
= local range2 = 10001 + 4999
- cd "$maindir/Historical Assets"
= cd "C:/Users/soha_/Desktop/MorningStar/1-Inputs/Historical Assets"
C:\Users\soha_\Desktop\MorningStar\1-Inputs\Historical Assets
- forvalues i=`range1'/`range2' {
= forvalues i=10001/15000 {
- use "`files'" in `i', clear
= use "C:\Users\soha_\AppData\Local\Temp\ST_00000001.tmp" in 10001, clear
observation numbers out of range
  local f = dirname + "/" + filename
  local filename = filename
  insheet using "`f'", clear
  local name: subinstr local filename ".csv" ""
  save "Assets_dta/`name'.dta", replace
  }
  }
Thanks,
Soha