%PDF-
%PDF-
Mini Shell
Mini Shell
<?
/**
* This class is for internal use only, not a part of public API.
* It can be changed at any time without notification.
*
* @access private
*/
/*
//HOWTO call:
$compiler = new Compiler\Compiler(array(
'workDir' => 'locations_data/',
'grabbedStuffDir' => 'locations_data/grabber/output/',
'grabber' => $grabber,
'includeYaInfo2Name' => false,
'fiasAddrobjFile' => 'locations_data/addrobj.xml'
));
$compiler->compile();
*/
namespace Bitrix\Sale\Location\Import\Compiler;
use Bitrix\Sale\Location\Import;
final class Compiler {
const OUTPUT_DIR = 'compiled/';
const MAPS_DIR = 'maps/';
const INPUT_DIR = 'output/';
const OUTPUT_FILE = 'output.txt';
const CODE_LENGTH = 10;
const TMP_DATA_DIR = 'tmp/';
const STATIC_CSV_DIR = 'static_csv/';
const GROUP_FILE = 'typegroup.csv';
const KAZAKHSTAN_SOURCE = 'ukrain_kazakhstan/loc_kz.csv';
const USA_SOURCE = 'ukrain_kazakhstan/loc_usa.csv';
const WORLD_SOURCE = 'ukrain_kazakhstan/loc_cntr.csv';
const CIS_SOURCE = 'ukrain_kazakhstan/loc_ussr_cut.csv';
const RUSSIA_YANDEX_CODE = 225;
const BELORUSSIA_YANDEX_CODE = 149;
const UKRAIN_YANDEX_CODE = 187;
const KAZAKHSTAN_YANDEX_CODE = 159;
const SOURCE_YANDEX = 'Y'; // got from yandex
const SOURCE_FIAS = 'F'; // got from fias
const SOURCE_UKRAIN = 'U'; // data sent by the office in ukrain
const SOURCE_LEGACY = 'L'; // got from old location files
const TMP_DATA_RUS_EXPORT_INDEX = 'rus_exp_index';
const TMP_DATA_RUS_GLOBAL_INDEX = 'glob_exp_index';
private $queue = false;
private $typeMap = array(
'SUBJECT_FEDERATION' => 'REGION'
);
private $headers = array(
'SHORT' => array('CODE', 'PARENT_CODE', 'NAME.RU.NAME', 'NAME.EN.NAME', 'NAME.UA.NAME'),
'LONG' => array('CODE', 'PARENT_CODE', 'TYPE_CODE', 'NAME.RU.NAME', 'NAME.EN.NAME', 'NAME.UA.NAME', 'LONGITUDE', 'LATITUDE', 'EXT.YAMARKET.0', 'EXT.ZIP.0'/*, 'EXT.ZIP.1', 'EXT.ZIP.2', 'EXT.ZIP.3'*/),
'GROUP_FILE' => array('CODE', 'TYPES')
);
private $typeGroups = array(
'LAYOUT' => array(
'CODE' => 'LAYOUT',
'TYPES' => array('COUNTRY', 'COUNTRY_DISTRICT', 'REGION'),
'HEADER' => 'LONG',
'FILE_NAME_TEMPLATE' => 'layout.csv'
),
/*
'SELECTABLE' => array(
'TYPES' => array('COUNTRY', 'COUNTRY_DISTRICT', 'REGION'),
'HEADER' => 'LONG',
'FILE_NAME_TEMPLATE' => 'selectable.csv'
),
*/
'AREAS' => array(
'CODE' => 'AREAS',
'TYPES' => array('CITY', 'SUBREGION', 'VILLAGE'/*, 'CITY_DISTRICT', 'METRO_STATION', 'OTHER'*/),
'PARENT' => 'LAYOUT',
'HEADER' => 'LONG',
'FILE_NAME_TEMPLATE' => '%BASE_PARENT_ITEM_CODE%_%CODE%.csv'
),
'STREETS' => array(
'CODE' => 'STREETS',
'TYPES' => array('STREET'),
'PARENT' => 'LAYOUT',
'HEADER' => 'LONG',
'FILE_NAME_TEMPLATE' => '%BASE_PARENT_ITEM_CODE%_%CODE%.csv'
)
);
private $fiasToBaseType = array(
'COUNTRY' => array(),
'COUNTRY_DISTRICT' => array(
'округ' => array('R' => 'округ', 'U' => true),
),
'REGION' => array(
'АО' => array('R' => 'автономный округ', 'U' => true),
'Аобл' => array('R' => 'автономная область', 'U' => true),
'край' => array('R' => 'край', 'U' => true),
'обл' => array('R' => 'область', 'U' => true),
'Респ' => array('R' => 'республика', 'U' => true),
'Чувашия' => array('R' => 'республика', 'U' => true)
),
'SUBREGION' => array(
'р-н' => array('R' => 'район', 'U' => true),
'улус' => array('R' => 'улус', 'U' => true),
'у' => array('R' => 'улус', 'U' => true),
),
'CITY' => array(
'г' => array('R' => 'город', 'U' => true),
),
'VILLAGE' => array(
'пгт' => array('R' => 'посёлок городского типа', 'U' => true),
'п' => array('R' => 'посёлок', 'U' => true),
'дп' => array('R' => 'дачный посёлок', 'U' => true),
'с/п' => array('R' => 'сельское поселение', 'U' => true),
'аал' => array('R' => 'аал', 'U' => true),
'аул' => array('R' => 'аул', 'U' => true),
'арбан' => array('R' => 'арбан', 'U' => true),
'д' => array('R' => 'деревня', 'U' => true),
'нп' => array('R' => 'населённый пункт', 'U' => true),
'сл' => array('R' => 'слобода', 'U' => true),
'х' => array('R' => 'хутор', 'U' => true),
'ферма' => array('R' => 'ферма', 'U' => true),
'с' => array('R' => 'село', 'U' => true),
'рп' => array('R' => 'рабочий посёлок', 'U' => true),
'ст' => array('R' => 'станция', 'U' => true),
'п/ст' => array('R' => 'посёлок', 'U' => true),
'ст-ца' => array('R' => 'станица', 'U' => true),
'кп' => array('R' => 'курортный поселок', 'U' => true),
'ж/д_ст' => array('R' => 'железнодорожная станция'),
'тер' => array('R' => 'территория'),
'остров' => array('R' => 'остров'),
'мкр' => array('R' => 'микрорайон', 'U' => true),
'с/с' => array('R' => 'сельсовет', 'U' => true),
'п/о' => array('R' => 'почтовое отделение', 'U' => true),
'м' => array('R' => 'местечко', 'U' => true),
'с/мо' => array('R' => 'смо', 'U' => true),
'жилрайон' => array('R' => 'жилрайон', 'U' => true),
'массив' => array('R' => 'массив'),
'ж/д_оп' => array('R' => 'ж/д остановка'),
'с/а' => array('R' => 'сельская администрация', 'U' => true),
'п/р' => array('R' => 'планировочный район'),
'ж/д_рзд' => array('R' => 'ж/д разъезд'),
'снт' => array('R' => 'снт', 'U' => true),
'с/о' => array('R' => 'сельский округ'),
'заимка' => array('R' => 'заимка'),
'городок' => array('R' => 'городок', 'U' => true)
),
//'CITY_DISTRICT' => array(),
//'METRO_STATION' => array(),
'STREET' => array(
'ул' => array('R' => 'улица', 'U' => true),
'кв-л' => array('R' => 'квартал', 'U' => true),
'аллея' => array('R' => 'аллея', 'U' => true),
'вал' => array('R' => 'вал', 'U' => true),
'въезд' => array('R' => 'въезд', 'U' => true),
'наб' => array('R' => 'набережная', 'U' => true),
'пер' => array('R' => 'переулок', 'U' => true),
'пл' => array('R' => 'площадь', 'U' => true),
'пр-кт' => array('R' => 'проспект', 'U' => true),
'проезд' => array('R' => 'проезд', 'U' => true),
'проулок' => array('R' => 'проулок', 'U' => true),
'рзд' => array('R' => 'разъезд', 'U' => true),
'сад' => array('R' => 'сад', 'U' => true),
'сквер' => array('R' => 'сквер', 'U' => true),
'спуск' => array('R' => 'спуск', 'U' => true),
'тоннель' => array('R' => 'тоннель', 'U' => true),
'тракт' => array('R' => 'тракт', 'U' => true),
'туп' => array('R' => 'тупик', 'U' => true),
'эстакада' => array('R' => 'эстакада', 'U' => true),
'б-р' => array('R' => 'бульвар', 'U' => true),
'бугор' => array('R' => 'бугор', 'U' => true),
'заезд' => array('R' => 'заезд', 'U' => true),
'канал' => array('R' => 'канал', 'U' => true),
'км' => array('R' => 'километр', 'U' => true),
'кольцо' => array('R' => 'кольцо', 'U' => true),
'парк' => array('R' => 'парк', 'U' => true),
'переезд' => array('R' => 'переезд', 'U' => true),
'стр' => array('R' => 'строение', 'U' => true),
'просек' => array('R' => 'просек', 'U' => true),
'ш' => array('R' => 'шоссе', 'U' => true),
'автодорога' => array('R' => 'дорога', 'U' => true)
)
);
private $forbiddenPathTypes = array(
'ул' => 1,
'пр-кт' => 1,
'проезд' => 1,
'ш' => 1,
'кв-л' => 1,
'км' => 1,
'просек' => 1,
'пер' => 1,
'б-р' => 1,
'наб' => 1,
'ж/д_будка' => 1,
'тракт' => 1,
'дор' => 1,
'рзд' => 1,
'пл' => 1,
'высел' => 1,
'сад' => 1,
'уч-к' => 1,
'промзона' => 1,
'автодорога' => 1,
'ж/д_платф' => 1,
);
private $forbiddenPathIds = array(
'af7cdb7f-e47d-4f65-93d5-3a2b70a809ce' => true, // Боровой микрорайон should be placed inside village, not street
'762758bb-18b9-440f-bc61-8e1e77ff3fd8' => true, // московский посёлок cannot be inside московский город, those are the same
);
private $allowedFiasStats = array(
'ACTSTATUS' => array(
1 // актуальный
),
'LIVESTATUS' => array(
1 // жив!
),
'CURRSTATUS' => array(
0, // актуальный
51, // переподчинённый
)
);
private $filePools = array(
'ukrain_kazakhstan' => array(
'DIR' => 'ukrain_kazakhstan/'
),
// where we store clean and split fias data:
'fias_tree' => array(
'DIR' => 'fias_tree/'
),
// where we keep maps from yandex to fias (REGIONS, CITIES and VILLAGES)
'fias_yamarket_links' => array(
'DIR' => 'fias_yamarket_links/'
),
// where we keep result data
'assets' => array(
'DIR' => 'compiled/bundles/extended/'
),
// where we keep result data, only for russia
'assets_standard' => array(
'DIR' => 'compiled/bundles/standard/'
),
// where we keep result data
'demo' => array(
'DIR' => 'demo/'
),
);
private $workDir = '';
private $grabbedStuffDir = '';
private $yaIdType = array();
private $relations = array();
private $options = array();
// tree builder
private $data = array();
private $optionConvertNames = false;
private $sysMaps = array();
private $fiasCPath = array();
private $fiasDB = null;
private $eTreeDB = null;
private $eTreeDBRussia = null;
public function __construct($options)
{
$this->workDir = $options['workDir'];
$this->grabbedStuffDir = $options['grabbedStuffDir'];
$this->options = $options;
//if(!file_exists($_SERVER['DOCUMENT_ROOT'].$this->workDir.self::OUTPUT_DIR))
// mkdir($_SERVER['DOCUMENT_ROOT'].$this->workDir.self::OUTPUT_DIR, 0700, true);
foreach($this->typeGroups as $id => &$params)
$params['I_TYPES'] = array_flip($params['TYPES']);
foreach($this->fiasToBaseType as $type => $fTypes)
{
foreach($fTypes as $fType => $fReplace)
{
if(strlen($fType) && !empty($fReplace))
{
$this->sysMaps['FIAS2BASETYPE'][$fType] = $type;
$this->sysMaps['FIASTYPEREPLACE'][$fType] = $fReplace['R'];
}
}
}
foreach($this->typeGroups as $groupId => $group)
{
//$this->output($group);
foreach($group['TYPES'] as $type)
{
$this->sysMaps['BASETYPE2GROUP'][$type] = $groupId;
}
}
$this->cleanOutput();
}
public function compile()
{
// step 1: build main tree from yandex market data
//$this->buildMainTree();
##########################################################
#### MAP FIAS TO YANDEX
##########################################################
// step 2: processing huge fias file
//$this->splitFiasOnRegions(); // split huge fias file onto small pieces of bundles
/*
Handwriting:
move 0c5b2444-70a0-4932-980c-b4dc0d3f02b5;1;1;Москва;г; TO 29251dcf-00a1-4e34-98d4-5c47484a36d4.csv
move c2deb16a-0330-4f05-821f-1d09c93331e6;1;1;Санкт-Петербург;г;190000 TO 6d1ebb35-70c6-4129-bd55-da3969658f5d.csv
move 6fdecb78-893a-4e3f-a5ba-aa062459463b;1;1;Севастополь;г; TO bd8e6511-e4b9-4841-90de-6bbc231a789e.csv
*/
//$this->copyFias2DB(); // place fias to db
// step 3: map fias to yandex id. There were some handwriting to map files, so do not uncomment unless you want files to be overwritten
//$this->mapFiasRootV2(); // process fias root and find matches by regions
// then we have a little handwriting on rootv2.csv
//$this->mapFiasCities(); // find cities and villages matches
// again, handwriting here
##########################################################
#### FIAS 2 DB
##########################################################
/*
$this->copyFias2DB();
update b_tmp_fias set PARENTGUID = '29251dcf-00a1-4e34-98d4-5c47484a36d4' where AOGUID = '0c5b2444-70a0-4932-980c-b4dc0d3f02b5' and AOID = '5c8b06f1-518e-496e-b683-7bf917e0d70b';
update b_tmp_fias set PARENTGUID = '6d1ebb35-70c6-4129-bd55-da3969658f5d' where AOGUID = 'c2deb16a-0330-4f05-821f-1d09c93331e6' and AOID = 'aad1469e-54ff-4605-af4f-f016c75b84d2';
update b_tmp_fias set PARENTGUID = 'bd8e6511-e4b9-4841-90de-6bbc231a789e' where AOGUID = '6fdecb78-893a-4e3f-a5ba-aa062459463b' and AOID = '6fdecb78-893a-4e3f-a5ba-aa062459463b';
*/
##########################################################
#### MAKE EXPORT TABLE
##########################################################
/*
$res = YandexGeoCoder::query(array(
'query' => 'Ненецкий автономный округ Шойна село Школьная улица',
'kind' => YandexGeoCoder::KIND_STREET
));
*/
$this->output('Build main tree');
$this->buildMainTree();
$this->createExportTables();
/*
###################################################################
###################################################################
###################################################################
// making world
$this->eTreeDB->cleanup();
$this->eTreeDB->dropIndexes();
$this->output('Generate export tree: Belarus');
$this->generateExportTreeBelorussia();
$this->output('Generate export tree: Kazakhstan');
$this->generateExportTreeLegacy(self::KAZAKHSTAN_SOURCE);
$this->output('Generate export tree: Ukrain');
$this->generateExportTreeUkrain();
$this->output('Generate export tree: USA');
$this->generateExportTreeUSA();
$this->output('Generate export tree: World Countries');
$this->generateExportTreeWorld();
$this->output('Generate export tree: EX-CIS');
$this->generateExportTreeLegacy(self::CIS_SOURCE);
###################################################################
###################################################################
###################################################################
$this->output('Last occupied: '.$this->eTreeDB->getLastOccupiedCode());
$this->output('Next free: '.$this->eTreeDB->getNextFreeCode());
// making russia
$this->eTreeDBRussia->cleanup();
$this->eTreeDBRussia->dropIndexes();
$this->output('Generate export tree: Russia');
$this->generateExportTreeRussia();
$this->output('Last occupied: '.$this->eTreeDBRussia->getLastOccupiedCode());
$this->output('Next free: '.$this->eTreeDBRussia->getNextFreeCode());
###################################################################
###################################################################
###################################################################
*/
$this->restoreExportTablesIndexes();
$this->output('Build export files');
/*
$this->cleanPoolDir('assets');
$this->eTreeDB->walkInDeep(array($this, 'generateExportFilesFromTableBundle')); // world
$this->eTreeDBRussia->walkInDeep(array($this, 'generateExportFilesFromTableBundle')); // russia
*/
$this->cleanPoolDir('assets_standard');
$this->eTreeDB->walkInDeep(array(
'ITEM' => array($this, 'generateExportFilesFromTableBundle_Standard')
)); // world
$this->eTreeDBRussia->walkInDeep(array(
'ITEM' => array($this, 'generateExportFilesFromTableBundle_Standard_YandexOnly')
)); // russia
/*
// types by groups
$this->makeTypeGroupFile(self::GROUP_FILE);
$this->copyStaticCSV();
*/
###################################################################
###################################################################
###################################################################
/*
$this->output('Build demo files');
$this->cleanPoolDir('demo');
$this->eTreeDB->walkInDeep(array($this, 'generateDemoFilesWorld'), array('VILLAGE' => 1)); // world
$this->eTreeDBRussia->walkInDeep(array($this, 'generateDemoFilesRussia'), array('VILLAGE' => 1)); // world
*/
/*
добавить сюда генерацию файла country_codes.php с содержимым:
<?
$LOCALIZATION_COUNTRY_CODE_MAP = array(
'ru' => '0000028023',
'ua' => '0000000364',
'kz' => '0000000276',
'bl' => '0000000001'
);
этот файл потом идёт в мастер установки интернет-магазина, вместе с демо-данными, types.csv и externalservice.csv
*/
$this->output('DONE');
}
#######################################################
### ABOUT EXPORT TABLE
#######################################################
private $allowedForDemo = array('COUNTRY' => 1, 'COUNTRY_DISTRICT' => 1, 'REGION' => 1, 'SUBREGION' => 1, 'CITY' => 1);
private $demoCategory = false;
public function generateDemoFilesWorld($item, $table)
{
if(!isset($this->allowedForDemo[$item['TYPE_CODE']]))
return;
if($item['TYPE_CODE'] == 'COUNTRY')
{
$this->addItemToCSV('world', 'demo', $item);
// this part must not depend on codes, which may flow left and right. in future we may add
// some markers like "is_ukrain" or "is_russia" etc to database instead of relying on names
if($item['NAME'] == 'Україна')
$this->demoCategory = 'ukrain';
elseif($item['NAME'] == 'Казахстан')
$this->demoCategory = 'kazakhstan';
elseif($item['NAME'] == 'Беларусь')
$this->demoCategory = 'belarus';
elseif($item['NAME'] == 'США')
$this->demoCategory = 'usa';
else
$this->demoCategory = false;
}
//$this->output($table->getWalkPathString());
//$this->output($this->demoCategory);
if($this->demoCategory !== false)
$this->addItemToCSV($this->demoCategory, 'demo', $item);
}
public function generateDemoFilesRussia($item, $table)
{
if(!isset($this->allowedForDemo[$item['TYPE_CODE']]))
return;
if($item['TYPE_CODE'] == 'COUNTRY')
$this->addItemToCSV('world', 'demo', $item);
$this->addItemToCSV('russia', 'demo', $item);//done
}
private function createExportTables()
{
$this->eTreeDB = new Db\ExportTreeTable();
$this->eTreeDB->create();
$this->eTreeDBRussia = new Db\ExportTreeRussiaTable();
$this->eTreeDBRussia->create();
}
private function cleanUpExportTables()
{
$this->eTreeDB->cleanup();
$this->eTreeDB->dropIndexes();
$this->eTreeDBRussia->cleanup();
$this->eTreeDBRussia->dropIndexes();
}
private function restoreExportTablesIndexes()
{
$this->eTreeDB->restoreIndexes();
$this->eTreeDBRussia->restoreIndexes();
}
private function copyStaticCSV()
{
$workDir = $_SERVER['DOCUMENT_ROOT'].'/'.$this->workDir;
system('cp '.$workDir.self::STATIC_CSV_DIR.'externalservice.csv '.$workDir.'/'.self::OUTPUT_DIR);
system('cp '.$workDir.self::STATIC_CSV_DIR.'type.csv '.$workDir.'/'.self::OUTPUT_DIR);
}
private $currentParentGroup = '';
private function addItemToCSV($fName, $group, $item)
{
$data = array(
'CODE' => $item['CODE'],
'PARENT_CODE' => $item['PARENT_CODE'],
'TYPE_CODE' => $item['TYPE_CODE']
);
$data['NAME.RU.NAME'] = '';
$data['NAME.EN.NAME'] = '';
$data['NAME.UA.NAME'] = '';
$name = unserialize($item['LANGNAMES']);
foreach($name as $lid => $values)
{
foreach($values as $i => $val)
$data['NAME.'.$lid.'.'.$i] = $val;
}
$data['EXT.YAMARKET.0'] = '';
$data['EXT.ZIP.0'] = '';
$externals = unserialize($item['EXTERNALS']);
if(!empty($externals))
{
foreach($externals as $type => $values)
{
if(is_array($values))
{
foreach($values as $i => $val)
$data['EXT.'.$type.'.'.$i] = $val;
}
}
}
$data['LONGITUDE'] = $item['LONGITUDE'];
$data['LATITUDE'] = $item['LATITUDE'];
/*
$this->output($data);
$this->output($group);
$this->output($fName);
*/
$this->putToFile2(
$data,
$group,
$fName,
true
);
}
public function generateExportFilesFromTableBundle($item, $table)
{
if(in_array($item['TYPE_CODE'], $this->typeGroups['LAYOUT']['TYPES']))
$this->currentParentGroup = $item['CODE'];
########################################################
########################################################
########################################################
$cat = $this->sysMaps['BASETYPE2GROUP'][$item['TYPE_CODE']];
$fName = $this->typeGroups[$cat]['FILE_NAME_TEMPLATE'];
$fName = str_replace(array(
'%BASE_PARENT_ITEM_CODE%',
'%CODE%',
'.csv'
), array(
$cat == 'LAYOUT' ? '' : $this->currentParentGroup,
ToLower($cat),
''
), $fName);
$this->addItemToCSV($fName, 'assets', $item);
########################################################
########################################################
########################################################
}
public function generateExportFilesFromTableBundle_Standard($item, $table)
{
if(in_array($item['TYPE_CODE'], $this->typeGroups['LAYOUT']['TYPES']))
$this->currentParentGroup = $item['CODE'];
########################################################
########################################################
########################################################
$cat = $this->sysMaps['BASETYPE2GROUP'][$item['TYPE_CODE']];
$fName = $this->typeGroups[$cat]['FILE_NAME_TEMPLATE'];
$fName = str_replace(array(
'%BASE_PARENT_ITEM_CODE%',
'%CODE%',
'.csv'
), array(
$cat == 'LAYOUT' ? '' : $this->currentParentGroup,
ToLower($cat),
''
), $fName);
$this->addItemToCSV($fName, 'assets_standard', $item);
########################################################
########################################################
########################################################
}
public function generateExportFilesFromTableBundle_Standard_YandexOnly($item, $table)
{
if(in_array($item['TYPE_CODE'], $this->typeGroups['LAYOUT']['TYPES']))
$this->currentParentGroup = $item['CODE'];
########################################################
########################################################
########################################################
if($item['TYPE_CODE'] == 'VILLAGE' && strpos($item['EXTERNALS'], 'YAMARKET') === false/*not from yandex database*/)
{
//$this->output($item['NAME'].' skipped');
return false;
}
$cat = $this->sysMaps['BASETYPE2GROUP'][$item['TYPE_CODE']];
$fName = $this->typeGroups[$cat]['FILE_NAME_TEMPLATE'];
$fName = str_replace(array(
'%BASE_PARENT_ITEM_CODE%',
'%CODE%',
'.csv'
), array(
$cat == 'LAYOUT' ? '' : $this->currentParentGroup,
ToLower($cat),
''
), $fName);
$this->addItemToCSV($fName, 'assets_standard', $item);
########################################################
########################################################
########################################################
return true;
}
private function generateExportTreeRussia()
{
$this->fiasDB = new Db\FiasTable();
$this->eTreeDBRussia->dropCodeIndex();
$this->eTreeDBRussia->setExportOffset(intval($this->eTreeDB->getNextFreeCode())); // start where the previous table ended
// get yandex regions
$regions = $this->readFiasRootMapV2();
// add Russia (country), districts and regions, that are taken from yandex
$this->generateExportTreePutRussiaBundle(array(self::RUSSIA_YANDEX_CODE), $regions, 0);
// add all precious content from fias: subregions, cities, villages, streets
$this->generateExportTreePutRussiaInner();
$this->eTreeDBRussia->doneInsert();
$this->eTreeDBRussia->switchIndexes(true);
}
private function getYandexToFiasCityMap($yandexRegionId)
{
$result = array();
$fias2yandex = $this->getDataFromCSV('fias_yamarket_links', 'region_'.$yandexRegionId);
foreach($fias2yandex as $map)
{
unset($map['HZ']); // csv viewer would crash without this key (wonder why)
$result[$map['AOGUID']] = $map;
}
return $result;
}
private function generateExportTreePutRussiaInner()
{
// get yandex-to-fias region code map
$links = $this->getFias2YamarketRootLinks(false, true);
$i = -1;
// for each region we must dump its content
// $yRId is a yandex id for the target region
// $fiasRegions is one (mostly) or several corresponding "regions" from fias
foreach($links as $yRId => $fiasRegions)
{
$i++;
// get yandex-to-fias city code map for the current region
$this->fias2yandexCityMap = $this->getYandexToFiasCityMap($yRId);
$this->currentRegion = $this->mapETCodeAsYandex($yRId);
$this->eTreeDBRussia->dropCodeIndex(); // drop previous region index
foreach($fiasRegions as $regionGuid)
{
$this->fiasPath = array();
$this->generateExportTreePutRussiaInnerBundle($regionGuid);
}
//break;
}
}
private function checkIsAllowedCityVillage($item)
{
$baseType = $this->sysMaps['FIAS2BASETYPE'][$item['SHORTNAME']];
//if($baseType == 'CITY' || $baseType == 'VILLAGE')
// $this->output('Doubtfull city\village: '.$item['FORMALNAME'].' ('.$item['SHORTNAME'].')');
$skip = array('5544bf6a-0ec1-4b5f-bbc5-49294f71de16'/*тягловая подстанция*/, '22a77f13-3764-41dc-aa23-db680b03ef5d'/*6 км АЗС*/, '3f2ab130-274e-4fd6-b611-a94467b04f57'/*Велтон Парк duplicate*/, '762758bb-18b9-440f-bc61-8e1e77ff3fd8', /*Московский посёлок, not exists*/);
return ($baseType == 'CITY' || $baseType == 'VILLAGE') &&
(
$this->fiasToBaseType[$baseType][$item['SHORTNAME']]['U'] /*code is in a list of allowed types for export*/
||
isset($this->fias2yandexCityMap[$item['AOGUID']] /*code is present in yandex2fias city map*/
) &&
!in_array($item['AOGUID'], $skip) // not one of those forbidden broken items
);
}
private function checkIsAllowedStreet($item)
{
$baseType = $this->sysMaps['FIAS2BASETYPE'][$item['SHORTNAME']];
return ($baseType == 'STREET' && $this->fiasToBaseType[$baseType][$item['SHORTNAME']]['U']); /*street code is in a list of allowed types for export*/
}
public function generateExportTreePutRussiaFiasPathCutForbidden($path)
{
/*
$object = $path[count($path) - 1];
if($object['AOGUID'] == 'da99f366-1a88-43b1-9aa8-c1b66334c97f')
{
$found = true;
\_print_r('Wow: da99f366-1a88-43b1-9aa8-c1b66334c97f');
\_print_r($object);
}
*/
//$path = array_reverse($path);
$newPath = array();
$lastValidId = false;
$neepPasteLastValid = false;
foreach($path as $item)
{
if(isset($this->forbiddenPathTypes[$item['SHORTNAME']]) || isset($this->forbiddenPathIds[$item['AOGUID']]) || $this->checkIsAllowedStreet($item['SHORTNAME']))
{
$neepPasteLastValid = true;
continue;
}
else
{
if($lastValidId !== false && $neepPasteLastValid)
{
$item['PARENTGUID'] = $lastValidId;
$neepPasteLastValid = false;
}
$lastValidId = $item['AOGUID'];
}
$newPath[] = $item;
}
/*
if($found)
{
\_print_r('Path is now:');
\_print_r($newPath);
die();
}
*/
return $newPath;
}
private function generateExportTreePutRussiaFiasPath($targetItem)
{
// pre-process, cut off unwanted types (actually, streets)
$newPath = $this->generateExportTreePutRussiaFiasPathCutForbidden($this->fiasPath);
$i = -1;
foreach($newPath as $item)
{
$i++;
// external data
$externals = array();
if(strlen($item['POSTALCODE']))
$externals['ZIP'][] = $item['POSTALCODE'];
if(isset($this->fias2yandexCityMap[$item['AOGUID']]))
$externals['YAMARKET'][] = $this->fias2yandexCityMap[$item['AOGUID']]['ID'];
// type and name
$itemType = $item['SHORTNAME'];
$baseType = $this->sysMaps['FIAS2BASETYPE'][$itemType];
$typeNameReplace = $baseType != 'CITY' ? $this->sysMaps['FIASTYPEREPLACE'][$itemType] : ''; // replace base type (e.g. "п" => "посёлок", "д" => "деревня", ...)
$name = trim($item['FORMALNAME']).(strlen($typeNameReplace) ? ' '.$typeNameReplace : '');
$this->eTreeDBRussia->insert(array(
'TYPE_CODE' => $baseType,
'FIAS_TYPE' => $itemType,
'NAME' => $name,
'LANGNAMES' => array('RU' => array('NAME' => $name)),
'EXTERNALS' => $externals,
'SOURCE' => self::SOURCE_FIAS,
'SYS_CODE' => $this->mapETCodeAsFias($item['AOGUID']),
'PARENT_SYS_CODE' => $i ? $this->mapETCodeAsFias($item['PARENTGUID']) : $this->currentRegion
));
}
}
private function generateExportTreePutRussiaStreets($parentGuid)
{
if(!strlen($parentGuid))
return;
$res = $this->fiasDB->getActualChildren($parentGuid);
while($item = $res->fetch())
{
$itemType = $item['SHORTNAME'];
if($this->checkIsAllowedStreet($item))
{
$externals = array();
if(strlen($item['POSTALCODE']))
$externals['ZIP'][] = $item['POSTALCODE'];
$baseType = $this->sysMaps['FIAS2BASETYPE'][$itemType];
$name = trim($item['FORMALNAME']).' '.$this->sysMaps['FIASTYPEREPLACE'][$itemType];
$this->eTreeDBRussia->insert(array(
'TYPE_CODE' => $baseType,
'FIAS_TYPE' => $itemType,
'NAME' => $name,
'LANGNAMES' => array('RU' => array('NAME' => $name)),
'EXTERNALS' => $externals,
'SOURCE' => self::SOURCE_FIAS,
'SYS_CODE' => $this->mapETCodeAsFias($item['AOGUID']),
'PARENT_SYS_CODE' => $this->mapETCodeAsFias($parentGuid),
));
}
}
}
private function generateExportTreePutRussiaInnerBundle($parentGuid)
{
if(!strlen($parentGuid))
return;
$res = $this->fiasDB->getActualChildren($parentGuid);
while($item = $res->fetch())
{
$item['PARENTGUID'] = $parentGuid;
array_push($this->fiasPath, $item);
if($this->checkIsAllowedCityVillage($item)) // check if this is a village or city or what else we should add to export
{
//$this->output('Allowed city\village '.$item['FORMALNAME']);
$this->generateExportTreePutRussiaFiasPath($item); // ALSO store intermediate locations from current region-to-city(village, etc) being stored
$this->generateExportTreePutRussiaStreets($item['AOGUID']); // store all streets of current village\city\...
}
$this->generateExportTreePutRussiaInnerBundle($item['AOGUID']);
array_pop($this->fiasPath);
}
}
private function generateExportTreePutRussiaBundle($bundle, $regions, $dl = 0)
{
foreach($bundle as $id)
{
$node = $this->data['TREES']['MAIN']['NODES'][$id];
$edges = array();
if(isset($this->data['TREES']['MAIN']['EDGES'][$id]))
$edges = $this->data['TREES']['MAIN']['EDGES'][$id];
if(in_array($node['TYPE_CODE'], array('COUNTRY', 'COUNTRY_DISTRICT', 'REGION')))
{
if($node['TYPE_CODE'] == 'REGION')
{
// get fias code & postal code, if any
$fNode = $this->fiasDB->getByAOGUID($regions[$node['ID']]['AOGUID']);
if(strlen($fNode['POSTALCODE']))
$node['EXT']['ZIP'][] = $fNode['POSTALCODE'];
$edges = array(); // no way farther
}
// set english name for Russia
if($node['TYPE_CODE'] == 'COUNTRY')
$node['NAME']['EN']['NAME'] = 'Russian Federation';
if($node['NAME']['RU']['NAME'] == 'Москва и Московская область')
$node['NAME']['RU']['NAME'] = 'Московская область';
if($node['NAME']['RU']['NAME'] == 'Санкт-Петербург и Ленинградская область')
$node['NAME']['RU']['NAME'] = 'Ленинградская область';
$this->eTreeDBRussia->insert(array(
'TYPE_CODE' => $node['TYPE_CODE'],
'NAME' => $node['NAME']['RU']['NAME'],
'LANGNAMES' => $node['NAME'],
'EXTERNALS' => $node['EXT'],
'SOURCE' => self::SOURCE_YANDEX,
'SYS_CODE' => $this->mapETCodeAsYandex($node['ID']),
'PARENT_SYS_CODE' => strlen($node['PARENT_ID']) && $dl > 0 ? $this->mapETCodeAsYandex($node['PARENT_ID']) : ''
));
if(!empty($edges))
$this->generateExportTreePutRussiaBundle($edges, $regions, $dl+1);
}
}
}
#########################
private function generateExportTreeWorld()
{
$this->eTreeDB->dropCodeIndex();
$this->eTreeDB->restoreExportOffset();
$this->eTreeDB->switchIndexes(false);
$csv = new Import\CSVReader();
$csv->loadFile($_SERVER['DOCUMENT_ROOT'].'/locations_data/'.self::WORLD_SOURCE);
$countries = array();
while($item = $csv->Fetch())
{
$item = explode(',', $item[0]);
if(!isset($item[1])) // its a language marker
continue;
// exclude the following countries, kz we got an extended file for them
if(in_array($item[2], array('USA', 'Kazakhstan', 'Ukraine', 'Byelorussia', 'Russian Federation', 'Azerbaijan', 'Estonia', 'Georgia', 'Latvia', 'Lithuania', 'Moldavia', 'Turkmenistan', 'Armenia', 'Tadjikistan', 'Uzbekistan')))
continue;
$id = implode(':', $item);
$countries[] = $item[2].' - '.$item[4];
$this->eTreeDB->insert(array(
'TYPE_CODE' => 'COUNTRY',
'NAME' => $item[4],
'LANGNAMES' => serialize(array(
'RU' => array('NAME' => $item[4]),
'EN' => array('NAME' => $item[2])
)),
'EXTERNALS' => '',
'SOURCE' => self::SOURCE_LEGACY,
'SYS_CODE' => $this->mapETCodeAsLegacy($id),
'PARENT_SYS_CODE' => ''
));
}
$this->eTreeDB->doneInsert();
$this->eTreeDB->switchIndexes(true);
}
#########################
private function generateExportTreeLegacy($source)
{
$this->eTreeDB->dropCodeIndex();
$this->eTreeDB->restoreExportOffset();
$this->eTreeDB->switchIndexes(false);
$csv = new Import\CSVReader();
$csv->loadFile($_SERVER['DOCUMENT_ROOT'].'/locations_data/'.$source);
$lastOnes = array();
while($item = $csv->Fetch())
{
$item = explode(',', $item[0]);
if(!isset($item[1])) // its a language marker
continue;
if($item[0] == 'R')
{
$item[2] = preg_replace('# obl$#', ' region', $item[2]);
$item[4] = preg_replace('# обл$#', ' область', $item[4]);
}
$parentId = '';
if($item[0] == 'S')
$type = 'COUNTRY';
if($item[0] == 'R')
$type = 'REGION';
if($item[0] == 'T')
$type = 'CITY';
$id = implode(':', $item);
if($type == 'REGION')
$parentId = $lastOnes['COUNTRY'];
elseif($type == 'CITY')
$parentId = $lastOnes['PARENT'];
else
$parentId = '';
if($type != 'CITY')
{
$lastOnes[$type] = $id;
$lastOnes['PARENT'] = $id;
}
$this->eTreeDB->insert(array(
'TYPE_CODE' => $type,
'NAME' => $item[4],
'LANGNAMES' => serialize(array(
'RU' => array('NAME' => $item[4]),
'EN' => array('NAME' => $item[2])
)),
'EXTERNALS' => '',
'SOURCE' => self::SOURCE_LEGACY,
'SYS_CODE' => $this->mapETCodeAsLegacy($id),
'PARENT_SYS_CODE' => strlen($parentId) ? $this->mapETCodeAsLegacy($parentId) : ''
));
}
$this->eTreeDB->doneInsert();
$this->eTreeDB->switchIndexes(true);
}
#########################
private function generateExportTreeUSA()
{
$this->eTreeDB->dropCodeIndex();
$this->eTreeDB->restoreExportOffset();
$this->eTreeDB->switchIndexes(false);
$csv = new Import\CSVReader();
$csv->loadFile($_SERVER['DOCUMENT_ROOT'].'/locations_data/'.self::USA_SOURCE);
$lastOnes = array();
while($item = $csv->Fetch())
{
$item = explode(',', $item[0]);
if(!isset($item[1])) // its a language marker
continue;
$parentId = '';
if($item[0] == 'S')
$type = 'COUNTRY';
if($item[0] == 'R')
$type = 'REGION';
if($item[0] == 'T')
$type = 'CITY';
$id = implode(':', $item);
if($type == 'REGION')
$parentId = $lastOnes['COUNTRY'];
elseif($type == 'CITY')
$parentId = $lastOnes['PARENT'];
else
$parentId = '';
if($type != 'CITY')
{
$lastOnes[$type] = $id;
$lastOnes['PARENT'] = $id;
}
if($item['2'] == 'USA')
$item['4'] = 'США';
$this->eTreeDB->insert(array(
'TYPE_CODE' => $type,
'NAME' => $item[4],
'LANGNAMES' => serialize(array(
'RU' => array('NAME' => $item[4]),
'EN' => array('NAME' => $item[2])
)),
'EXTERNALS' => '',
'SOURCE' => self::SOURCE_LEGACY,
'SYS_CODE' => $this->mapETCodeAsLegacy($id),
'PARENT_SYS_CODE' => strlen($parentId) ? $this->mapETCodeAsLegacy($parentId) : ''
));
}
$this->eTreeDB->doneInsert();
$this->eTreeDB->switchIndexes(true);
}
#########################
private function generateExportTreeBelorussia()
{
$this->eTreeDB->dropCodeIndex();
$this->eTreeDB->restoreExportOffset();
$this->eTreeDB->switchIndexes(false);
$this->generateExportTreePutBelorussiaBundle(array(self::BELORUSSIA_YANDEX_CODE));
$this->eTreeDB->doneInsert();
$this->eTreeDB->switchIndexes(true);
}
private function generateExportTreePutBelorussiaBundle($bundle)
{
foreach($bundle as $id)
{
$node = $this->data['TREES']['MAIN']['NODES'][$id];
$edges = array();
if(isset($this->data['TREES']['MAIN']['EDGES'][$id]))
$edges = $this->data['TREES']['MAIN']['EDGES'][$id];
// attach to belorussia its en-name
if($node['TYPE_CODE'] == 'COUNTRY')
$node['NAME']['EN']['NAME'] = 'Belarus';
// these two types are not allowed currently, because we do not have the corresponding types in fias
if($node['TYPE_CODE'] == 'METRO_STATION' || $node['TYPE_CODE'] == 'CITY_DISTRICT')
continue;
$this->eTreeDB->insert(array(
'TYPE_CODE' => $node['TYPE_CODE'],
'NAME' => $node['NAME']['RU']['NAME'],
'LANGNAMES' => serialize($node['NAME']),
'EXTERNALS' => serialize($node['EXT']),
'SOURCE' => self::SOURCE_YANDEX,
'SYS_CODE' => $this->mapETCodeAsYandex($node['ID']),
'PARENT_SYS_CODE' => strlen($node['PARENT_ID']) ? $this->mapETCodeAsYandex($node['PARENT_ID']) : ''
));
if(!empty($edges))
$this->generateExportTreePutBelorussiaBundle($edges);
}
}
#########################
private function generateExportTreeUkrain()
{
$this->eTreeDB->dropCodeIndex();
$this->eTreeDB->restoreExportOffset();
$cd2r = $this->getDataFromCSV('ukrain_kazakhstan', 'ukrain_district2region');
$tree = array();
$tree['NODES'][self::UKRAIN_YANDEX_CODE] = array(
'NAME' => array(
'RU' => array('NAME' => 'Украина'),
'UA' => array('NAME' => 'Україна'),
'EN' => array('NAME' => 'Ukraine')
),
'TYPE_CODE' => 'COUNTRY',
'SOURCE' => self::SOURCE_YANDEX,
'ID' => self::UKRAIN_YANDEX_CODE,
'EXT' => array(
'YAMARKET' => array(
self::UKRAIN_YANDEX_CODE
)
)
);
foreach($cd2r as $line)
{
// add country district
if(!isset($tree['NODES'][$line['CDID']]))
{
$tree['NODES'][$line['CDID']] = array(
'NAME' => $line['CDNAME'],
'TYPE_CODE' => 'COUNTRY_DISTRICT',
'SOURCE' => self::SOURCE_YANDEX,
'ID' => $line['CDID'],
'PARENT_ID' => self::UKRAIN_YANDEX_CODE,
'EXT' => array(
'YAMARKET' => array(
$line['CDID'] // country district id in file
),
)
);
}
$tree['EDGES'][self::UKRAIN_YANDEX_CODE][$line['CDID']] = true;
$regionId = 'r'.md5($line['RNAME']['UA']['NAME']);
if($line['RNAME']['UA']['NAME'] == 'Севастополь, Місто' || $line['RNAME']['UA']['NAME'] == 'Автономна Республіка Крим')
$source = self::SOURCE_UKRAIN;
else
$source = self::SOURCE_YANDEX;
if($line['RNAME']['UA']['NAME'] == 'Севастополь, Місто')
{
$typeCode = 'SUBREGION';
}
else
{
$typeCode = 'REGION';
}
// add region
if(!isset($tree['NODES'][$regionId]))
{
$tree['NODES'][$regionId] = array(
'NAME' => $line['RNAME'],
'TYPE_CODE' => $typeCode,
'SOURCE' => $source,
'ID' => $regionId,
'PARENT_ID' => $line['CDID'],
);
if($source == self::SOURCE_YANDEX)
$tree['NODES'][$regionId]['EXT']['YAMARKET'][] = $line['RID'];
}
$tree['EDGES'][$line['CDID']][$regionId] = true;
}
$res = $this->getDataFromCSV('ukrain_kazakhstan', 'ukrain');
foreach($res as $line)
{
if($line['REGION'] == 'Автономна республіка Крим')
$line['REGION'] = 'Автономна Республіка Крим';
$line['REGION'] = $this->mb_str_replace('Місто', 'місто', $line['REGION']);
$line['SUBREGION'] = $this->mb_str_replace('Місто', 'місто', $line['SUBREGION']);
$line['CITY'] = $this->mb_str_replace('Місто', 'місто', $line['CITY']);
$regionId = 'r'.md5($line['REGION']);
$subRegionId = 'sr'.md5($line['SUBREGION']);
$cityId = 'c'.md5($line['CITY']);
if($line['REGION'] != 'Севастополь, Місто')
{
if(!isset($tree['NODES'][$subRegionId]))
{
$tree['NODES'][$subRegionId] = array(
'NAME' => array('UA' => array('NAME' => $line['SUBREGION'])),
'TYPE_CODE' => 'SUBREGION',
'SOURCE' => self::SOURCE_UKRAIN,
'ID' => $subRegionId,
'PARENT_ID' => $regionId,
'EXT' => array()
);
if(!isset($tree['EDGES'][$regionId][$subRegionId]))
$tree['EDGES'][$regionId][$subRegionId] = true;
}
}
if(!isset($tree['NODES'][$cityId]))
{
$tree['NODES'][$cityId] = array(
'NAME' => array('UA' => array('NAME' => $line['CITY'])),
'TYPE_CODE' => 'CITY',
'SOURCE' => self::SOURCE_UKRAIN,
'ID' => $cityId,
'PARENT_ID' => $subRegionId,
'EXT' => array()
);
if(!isset($tree['EDGES'][$subRegionId][$cityId]))
$tree['EDGES'][$subRegionId][$cityId] = true;
}
}
foreach($tree['EDGES'] as $k => $edges)
$tree['EDGES'][$k] = array_keys($edges);
$this->data['TREES']['UKRAIN'] = $tree;
$this->generateExportTreePutUkrainBundle(array(self::UKRAIN_YANDEX_CODE));
unset($this->data['TREES']['UKRAIN']);
$this->eTreeDB->doneInsert();
}
private function generateExportTreePutUkrainBundle($bundle)
{
foreach($bundle as $id)
{
$node = $this->data['TREES']['UKRAIN']['NODES'][$id];
$edges = array();
if(isset($this->data['TREES']['UKRAIN']['EDGES'][$id]))
$edges = $this->data['TREES']['UKRAIN']['EDGES'][$id];
$this->eTreeDB->insert(array(
'TYPE_CODE' => $node['TYPE_CODE'],
'NAME' => $node['NAME']['UA']['NAME'],
'LANGNAMES' => serialize($node['NAME']),
'EXTERNALS' => serialize($node['EXT']),
'SOURCE' => $node['SOURCE'],
'SYS_CODE' => $this->mapETCodeAsUkrainian($node['ID']),
'PARENT_SYS_CODE' => strlen($node['PARENT_ID']) ? $this->mapETCodeAsUkrainian($node['PARENT_ID']) : ''
));
if(!empty($edges))
$this->generateExportTreePutUkrainBundle($edges);
}
}
private function mapETCodeAsYandex($code)
{
return 'Y_'.$code;
}
private function mapETCodeAsFias($code)
{
return 'F_'.$code;
}
private function mapETCodeAsUkrainian($name)
{
return 'U_'.md5($name);
}
private function mapETCodeAsLegacy($name)
{
return 'L_'.md5($name);
}
#######################################################
### ABOUT EXPORT TREE GENERATION
#######################################################
private function mapETCodeBySource($value, $source)
{
if($source == self::SOURCE_YANDEX)
return 'Y_'.$value;
if($source == self::SOURCE_FIAS)
return 'F_'.$value;
if($source == self::SOURCE_UKRAIN)
return 'U_'.md5($name);
if($source == self::SOURCE_KAZAKHSTAN)
return 'K_'.md5($name);
}
private function startExportFromScratch()
{
$this->cleanTemporalData(self::TMP_DATA_RUS_EXPORT_INDEX);
$this->cleanTemporalData(self::TMP_DATA_RUS_GLOBAL_INDEX);
$this->cleanPoolDir('assets');
}
private function generateExportTreeRussiaRoot()
{
$this->restoreTDRusExpIndex();
if(!empty($this->alreadyDumped))
return;
$regions = $this->readFiasRootMapV2();
$this->generateExportTreePutRussiaBundleOld(array(self::RUSSIA_YANDEX_CODE), $regions, 0);
$this->storeTemporalData(self::TMP_DATA_RUS_EXPORT_INDEX, $this->alreadyDumped);
}
private function restoreTDRusExpIndex()
{
if(!empty($this->alreadyDumped))
return;
$this->alreadyDumped = $this->getStoredTemporalData(self::TMP_DATA_RUS_EXPORT_INDEX);
}
private function storeTDGlobalExpIndex()
{
$this->storeTemporalData(self::TMP_DATA_RUS_GLOBAL_INDEX, array('I' => $this->exportOffset));
}
private function restoreTDGlobalExpIndex()
{
if($this->exportOffset == 0)
{
$data = $this->getStoredTemporalData(self::TMP_DATA_RUS_GLOBAL_INDEX);
$this->exportOffset = intval($data['I']);
}
}
private function generateExportTreePutRussiaBundleOld($bundle, $regions, $dl = 0)
{
foreach($bundle as $id)
{
$node = $this->data['TREES']['MAIN']['NODES'][$id];
$edges = array();
if(isset($this->data['TREES']['MAIN']['EDGES'][$id]))
$edges = $this->data['TREES']['MAIN']['EDGES'][$id];
if(in_array($node['TYPE_CODE'], array('COUNTRY', 'COUNTRY_DISTRICT', 'REGION')))
{
if($node['TYPE_CODE'] == 'REGION')
{
// get fias code & postal code, if any
//$node['EXT']['FIAS'][] = $regions[$node['ID']]['AOGUID'];
$fNode = $this->fiasGetByAOGUID($regions[$node['ID']]['AOGUID']);
if(strlen($fNode['POSTALCODE']))
$node['EXT']['ZIP'][] = $fNode['POSTALCODE'];
$edges = array(); // no way farther
}
$this->addItemToExportTree(array(
'ID' => $this->mapETCodeAsYandex($node['ID']),
'PARENT_ID' => strlen($node['PARENT_ID']) && $dl > 0 ? $this->mapETCodeAsYandex($node['PARENT_ID']) : '',
'TYPE' => $node['TYPE_CODE'],
'NAME' => $node['NAME'],
'EXTERNALS' => $node['EXT'],
'SOURCE' => self::SOURCE_YANDEX,
));
if(!empty($edges))
$this->generateExportTreePutRussiaBundle($edges, $regions, $dl+1);
}
}
}
private $fiasPath = array();
private $alreadyStoredPathItems = array();
private $fias2yandexMap = array();
private $alreadyDumped = array();
private $currentRegion = false;
private function generateExportTreeRussiaInner()
{
$links = $this->getFias2YamarketRootLinks(false, true);
//$this->output($links);
$i = -1;
foreach($links as $yRId => $regions)
{
$i++;
$this->restoreTDGlobalExpIndex();
$this->alreadyDumped = array();
$this->restoreTDRusExpIndex();
$this->fias2yandexMap = array();
$fias2yandex = $this->getDataFromCSV('fias_yamarket_links', 'region_'.$yRId);
foreach($fias2yandex as $map)
{
$this->fias2yandexMap[$map['AOGUID']] = $map;
}
unset($fias2yandex);
$this->currentRegion = $this->mapETCodeBySource($yRId, self::SOURCE_YANDEX);
foreach($regions as $regionId)
{
$this->fiasPath = array();
$this->output('CCCurrent REGION is '.$this->currentRegion);
//$this->output($this->alreadyDumped);
$this->generateExportTreeRussiaInnerBundle($regionId);
}
$this->storeTDGlobalExpIndex();
//break; // tmp
//if($i == 1)break;
}
}
private function generateExportTreeRussiaInnerBundle($parentGuid)
{
if(!strlen($parentGuid))
return;
$res = $this->getDataFromCSV('fias_tree', $parentGuid);
foreach($res as $item)
{
if($item['LIVESTATUS'] != '1' || $item['ACTSTATUS'] != '1')
continue;
$item['PARENT_ID'] = $parentGuid;
array_push($this->fiasPath, $item);
if($this->checkIsAllowedCityVillage($item['ID'], $item['TYPE']))
{
// ADD!!!
$this->storeCurrentFiasPath2();
$this->generateExportTreeRussiaInnerStreets($item['ID']);
}
$this->generateExportTreeRussiaInnerBundle($item['ID']);
array_pop($this->fiasPath);
}
}
private function addItemToExportTree($item)
{
$this->exportOffset++;
$this->output('ADD: '.$item['NAME']['RU']['NAME'].' '.$item['TYPE']);
//$this->output($item);
$cat = $this->sysMaps['BASETYPE2GROUP'][$item['TYPE']];
$fName = $this->typeGroups[$cat]['FILE_NAME_TEMPLATE'];
$header = $this->typeGroups[$cat];
$this->alreadyDumped[$item['ID']] = $this->exportOffset;
$parentCode = strlen($item['PARENT_ID']) ? $this->addLeadingZero($this->alreadyDumped[$item['PARENT_ID']], self::CODE_LENGTH) : '';
$data = array(
'CODE' => $this->addLeadingZero($this->exportOffset, self::CODE_LENGTH),
'PARENT_CODE' => $parentCode,
'TYPE_CODE' => $item['TYPE']
);
foreach($item['NAME'] as $lid => $values)
{
foreach($values as $i => $val)
$data['NAME.'.$lid.'.'.$i] = $val;
}
$data['EXT.YAMARKET.0'] = '';
$data['EXT.ZIP.0'] = '';
if(!empty($item['EXTERNALS']))
{
foreach($item['EXTERNALS'] as $type => $values)
{
foreach($values as $i => $val)
$data['EXT.'.$type.'.'.$i] = $val;
}
}
$parentGroupId = strlen($item['PARENT_GROUP_ID']) ? $this->addLeadingZero($this->alreadyDumped[$item['PARENT_GROUP_ID']], self::CODE_LENGTH) : '';
$data['LONGITUDE'] = '';
$data['LATITUDE'] = '';
//$this->output('PGID is set to '.$parentGroupId);
$fName = str_replace(array(
'%BASE_PARENT_ITEM_CODE%',
'%CODE%',
'.csv'
), array(
$parentGroupId,
ToLower($cat),
''
), $fName);
$this->output('PUT:');
$this->output($data);
$this->output('TO:');
$this->output($fName);
$this->putToFile2(
$data,
'assets',
$fName,
true
);
}
#######################################################
### ABOUT MAIN TREE
#######################################################
// map yandex cities to fias cities
public function buildMainTree()
{
$this->queue = false;
$this->data['TREES']['MAIN'] = array();
$this->data['MAPS'] = array();
$this->data['INDEXES'] = array();
$done = false;
while(!$done)
$done = $this->buildMainTreeNext();
// build name-path index for russia
$this->buildRussiaPathIndex($this->data['TREES']['MAIN']['EDGES'][self::RUSSIA_YANDEX_CODE], array());
}
private function buildMainTreeNext()
{
$next = $this->queueShift();
$bundle = $this->getBundleFromFile($next);
if(!empty($bundle))
{
foreach($bundle as $item)
{
$this->data['TREES']['MAIN']['NODES'][$item['ID']] = $item;
$parent = isset($item['PARENT_ID']) ? $item['PARENT_ID'] : 'ROOT';
$this->data['TREES']['MAIN']['EDGES'][$parent][] = $item['ID'];
if($item['CHILDREN_COUNT'] > 0)
$this->queue[] = $item['ID'];
}
}
return empty($this->queue);
}
#######################################################
### ABOUT FIAS PROCESS
#######################################################
// temporal function
private function checkFiasMaps()
{
$links = $res = $this->getDataFromCSV('fias_yamarket_links', 'rootv2');
$uTotal = 0;
foreach($links as $reg)
{
if($reg['CITIES_MAPPED'] != '1')
{
//$this->output($reg);
$rMap = $this->getDataFromCSV('fias_yamarket_links', 'region_'.$reg['YAMARKET']);
$this->output('============================ For: '.$reg['YAMARKET_NAME'].' '.$reg['YAMARKET']);
$unmapped = 0;
foreach($rMap as $map)
{
if(!strlen($map['AOGUID']))
{
$unmapped++;
$this->output($map);
$res = DB\FiasTable::getList(array('filter' => array(
'FORMALNAME' => $map['NAME'],
'!SHORTNAME' => array('ул', 'пер'),
'ACTSTATUS' => '1',
'LIVESTATUS' => '1'
)));
while($item = $res->fetch())
{
$this->output($item);
}
}
}
$this->output('Unmapped: '.$unmapped);
$uTotal += $unmapped;
}
}
$this->output('TOTAL: '.$uTotal);
}
private function showChildren($pId)
{
$res = DB\FiasTable::getList(array('filter' => array(
'PARENTGUID' => $pId,
'ACTSTATUS' => '1',
'LIVESTATUS' => '1'
)));
while($item = $res->fetch())
{
$this->output($item);
}
}
private function findPathes()
{
$pathes = array(
'aea7bac4-f9b4-4160-95f2-3d667b4d3f92', // отрадное, калиниградская область, 10857
'bda061ac-cbd0-4db8-8d18-69db43e76c2d', // отрадное, калиниградская область, 10857
'436b841d-a44f-431e-b1ec-7d76456d4a11', // отрадное, калиниградская область, 10857
'807943cc-31c0-4a86-a3fe-46d9262101e9', // дагомыс, краснодарский край, 10995
);
foreach($pathes as $id)
{
$this->fiasFindPath($id);
}
}
private function fiasFindPath($aoguid)
{
$pId = $aoguid;
while($pId && $res = DB\FiasTable::getList(array('filter' => array(
'AOGUID' => $pId,
'ACTSTATUS' => '1',
'LIVESTATUS' => '1'
)))->fetch())
{
$this->output($res);
if($res['PARENTGUID'])
$pId = $res['PARENTGUID'];
else
$pId = false;
}
}
private function fiasFind2()
{
$res = DB\FiasTable::getList(array('filter' => array(
'FORMALNAME' => array(
),
//'ACTSTATUS' => '1',
//'LIVESTATUS' => '1'
)));
while($item = $res->fetch())
{
$this->output($item);
}
}
private function fiasFind()
{
$findInFias = array(
//СС А РРР ГГГ ВВВ ППП УУУУ ЭЭЭЭ ЦЦЦ
'11 0 007 000 000 009 0000 0000 000', // кажым, коми, 10939
'35 0 003 000 000 001 0000 0000 000', // им бабушкина, волог. обл, 10853
'05 0 017 000 000 019 0000 0000 000', // ачи-су, дагестан, 11010
'20 0 028 000 000 001 0000 0000 000', // итум-кали, чечня, 11024
'16 0 022 000 000 023 0000 0000 000', // куланга, татарстан, 11119
'59 0 020 000 000 003 0000 0000 000', // гамово, пермский край, 11108
'56 0 000 000 000 002 0000 0000 000', // зато комаровский, оренбургская область, 11084
'86 0 003 000 000 031 0000 0000 000', // узюм-юрганская гкс, ханты-мансийскиий ао, 11193
'70 0 007 000 000 000 0000 0047 000', // игол, томская область, 11353
'27 0 009 000 000 001 0000 0000 000', // им полины осипенко, хабаровскйи край, 11457
'28 0 014 000 000 045 0000 0000 000', // свободный-21, амурская область, 11375
'14 0 010 000 000 001 0000 0000 000', // багатай, саха якутия, 11443
);
$res = DB\FiasTable::getList(array('filter' => array(
'CODE' => array_values($findInFias),
'ACTSTATUS' => '1',
'LIVESTATUS' => '1'
)));
while($item = $res->fetch())
{
$this->output($item);
}
}
private function fiasGetByAOGUID($fiasId)
{
return DB\FiasTable::getList(array('filter' => array(
'=AOGUID' => $fiasId,
)))->fetch();
}
/*
private function parseFiasCode($code)
{
//СС(0) А(1) РРР(2) ГГГ(3) ВВВ(4) ППП(5) УУУУ(6) ЭЭЭЭ(7) ЦЦЦ(8)
$code = explode(' ', $code);
return array(
'REGIONCODE' => $code[0],
'AREACODE' => $code[2],
'AUTOCODE' => $code[1],
'CITYCODE' => $code[3],
'CTARCODE' => $code[4],
'PLACECODE' => $code[5],
'STREETCODE' => $code[6],
'EXTRCODE' => $code[7],
'SEXTCODE' => $code[8]
);
}
private function checkFitItemByCode($code, $item)
{
$code = $this->parseFiasCode($code);
return (
$item['REGIONCODE'] == $code['REGIONCODE']
&&
$item['AREACODE'] == $code['AREACODE']
&&
$item['AUTOCODE'] == $code['AUTOCODE']
&&
$item['CITYCODE'] == $code['CITYCODE']
&&
$item['CTARCODE'] == $code['CTARCODE']
&&
$item['PLACECODE'] == $code['PLACECODE']
&&
$item['STREETCODE'] == $code['STREETCODE']
&&
$item['EXTRCODE'] == $code['EXTRCODE']
&&
$item['SEXTCODE'] == $code['SEXTCODE']
);
}
*/
private function mapFiasCities()
{
//$this->output($this->data['MAPS']['REGIONS']);
$links = $this->getFias2YamarketRootLinks();
//$this->output($links);
$typesToSearch = array('CITY', 'VILLAGE');
// for each region and city we choose the correpongind ones from fias, saving routes
foreach($links as $id => $fiasSource)
{
$region = $this->data['MAPS']['REGIONS'][$id];
$this->output('FOR region: '.$region);
$this->output($region);
$toBeFound = array(); // among all nodes get only ones with following types
$this->getMainTreeNodesOfType(array($id), $typesToSearch, $toBeFound);
//$this->output(count($toBeFound));
if(!empty($toBeFound)) // search them in fias prepared tree
{
foreach($fiasSource as $fiasId)
{
$this->output('In: '.$fiasId);
$this->walkFiasTreeAndKeepFollowing($fiasId, $toBeFound, $typesToSearch);
}
}
$this->cleanUpFile('fias_yamarket_links', 'region_'.$region['ID']);
foreach($toBeFound as $node)
{
$exactId = '';
$exactName = '';
$exactType = '';
$exactCode = '';
if(!empty($node['MATCH']))
{
$match = array_shift($node['MATCH']);
$exactId = $match['ID'];
$exactName = $match['NAME'];
$exactType = $match['TYPE'];
$item = $this->fiasGetByAOGUID($exactId);
if($item)
{
$exactCode = $item['CODE'];
}
else
$this->output('no record in fias for: '.$exactId);
}
$data = array(
'HZ' => 'libre',
'ID' => $node['ID'],
'NAME' => $node['NAME']['RU']['NAME'],
'AOGUID' => $exactId,
'FNAME' => $exactName,
'FTYPE' => $exactType,
'CODE' => $exactCode
);
for($i = 0; $i < 3; $i++)
{
$data['VAR_AOGUID_'.$i] = '';
$data['VAR_NAME_'.$i] = '';
$data['VAR_TYPE_'.$i] = '';
}
$i = 0;
if(!empty($node['MATCH']))
{
foreach($node['MATCH'] as $item)
{
$data['VAR_AOGUID_'.$i] = $item['ID'];
$data['VAR_NAME_'.$i] = $item['NAME'];
$data['VAR_TYPE_'.$i] = $item['TYPE'];
$i++;
}
}
if(!empty($node['POSSIBLE']))
{
foreach($node['POSSIBLE'] as $item)
{
$data['VAR_AOGUID_'.$i] = $item['ID'];
$data['VAR_NAME_'.$i] = $item['NAME'];
$data['VAR_TYPE_'.$i] = $item['TYPE'];
$i++;
}
}
$this->putToFile2(
$data,
'fias_yamarket_links',
'region_'.$region['ID'],
true
);
}
//break;//tmp
}
}
private function checkAllowedState($node)
{
return $node['ACTSTATUS'] == '1' && $node['LIVESTATUS'] == '1';
}
private function mb_str_replace($needle, $replace_text, $haystack)
{
return implode($replace_text, mb_split($needle, $haystack));
}
private function checkNamesEqual($one, $two)
{
// try trim-lc
$one = $this->makeNameIndexKey($one);
$two = $this->makeNameIndexKey($two);
if($one == $two)
return true;
// try ё => e
$one = $this->mb_str_replace('ё', 'е', $one);
$two = $this->mb_str_replace('ё', 'е', $two);
if($one == $two)
return true;
// try й => и
$one = $this->mb_str_replace('й', 'и', $one);
$two = $this->mb_str_replace('й', 'и', $two);
if($one == $two)
return true;
// there could be also multiple spaces between
//if(!($keptNode['NAME_I'] == $name || strpos($keptNode['NAME_I'], $name) !== false || strpos($name, $keptNode['NAME_I']) !== false))
// continue;
return false;
}
private function checkNamesAlmostEqual($one, $two)
{
$one = $this->makeNameIndexKey($one);
$two = $this->makeNameIndexKey($two);
if(strpos($one, $two) !== false || strpos($two, $one) !== false)
return true;
$one = $this->mb_str_replace('ё', 'е', $one);
$two = $this->mb_str_replace('ё', 'е', $two);
if(strpos($one, $two) !== false || strpos($two, $one) !== false)
return true;
$one = $this->mb_str_replace('й', 'и', $one);
$two = $this->mb_str_replace('й', 'и', $two);
if(strpos($one, $two) !== false || strpos($two, $one) !== false)
return true;
$one = preg_replace('#\s+-\s+#', '-', $one);
$two = preg_replace('#\s+-\s+#', '-', $two);
if(strpos($one, $two) !== false || strpos($two, $one) !== false)
return true;
return false;
}
private function walkFiasTreeAndKeepFollowing($node, &$toBeFound, $typesToSearch)
{
$bundle = $this->getDataFromCSV('fias_tree', $node);
if(is_array($bundle) && !empty($bundle))
{
//$this->output($bundle);
foreach($bundle as $node)
{
$name = $this->makeNameIndexKey($node['NAME']);
//$this->output($node);
// check if we need this
if(strlen($node['TYPE']) && isset($this->sysMaps['FIAS2BASETYPE'][$node['TYPE']]))
{
$type = $this->sysMaps['FIAS2BASETYPE'][$node['TYPE']];
if(in_array($type, $typesToSearch)) // type fits
{
// check if name fits too
foreach($toBeFound as &$keptNode)
{
if($keptNode['TYPE_CODE'] != $type)
continue;
if(!$this->checkAllowedState($node))
continue;
if($this->checkNamesEqual($name, $keptNode['NAME']['RU']['NAME']))
{
$keptNode['MATCH'][$node['AOGUID']] = $node;
}
elseif($this->checkNamesAlmostEqual($name, $keptNode['NAME']['RU']['NAME']))
{
$keptNode['POSSIBLE'][$node['AOGUID']] = $node;
}
}
}
}
$this->walkFiasTreeAndKeepFollowing($node['ID'], $toBeFound, $typesToSearch);
}
}
}
private function getFias2YamarketRootLinks($skipMapped = true, $skipCompiled = false)
{
$res = $this->getDataFromCSV('fias_yamarket_links', 'rootv2');
$result = array();
foreach($res as $reg)
{
if($skipMapped && $reg['CITIES_MAPPED'] == '1')
continue;
if($skipCompiled && $reg['COMPILED'] == '1')
continue;
$fias = array($reg['AOGUID']);
if(strlen($reg['ADDITIONAL']))
{
$variants = explode(', ', $reg['ADDITIONAL']);
foreach($variants as $var)
{
$id = explode(':', $var);
$fias[] = $id[0];
}
}
$result[$reg['YAMARKET']] = $fias;
}
return $result;
}
public function splitFiasOnRegions()
{
$this->cleanPoolDir('fias_tree');
$this->walkFias('fiasGotOneSplit');
}
public function copyFias2DB()
{
$this->fiasDB = new Db\FiasTable();
$this->fiasDB->create();
$this->fiasDB->switchIndexes(false);
$this->fiasDB->deleteAll();
$this->walkFias('fiasGotOneAdd2DB');
$this->fiasDB->doneInsert();
$this->fiasDB->switchIndexes(true);
}
/*
public function dropFiasTreeDuplicates()
{
foreach(new \DirectoryIterator($this->getPoolDirName('fias_tree')) as $file)
{
if($file->isDot() || $file->isDir())
continue;
$csv = $this->getDataFromCSV('fias_tree', str_replace('.csv', '', $file->getFilename()));
$index = array();
foreach($csv as $id => $line)
{
if(isset($index[$line['ID']]))
{
unset($csv[$id]);
continue;
}
$index[$line['ID']] = true;
}
$this->putDataToCSV($csv, 'fias_tree', str_replace('.csv', '', $file->getFilename()));
unset($index);
unset($csv);
}
}
*/
public function fiasGotOneSplit($data)
{
$item = $data['__ATTR'];
//$this->manageFiasPath($item);
//$this->output(str_repeat('-', intval($item['AOLEVEL']) - 1).$item['FORMALNAME']);
//$this->output($this->printCurrentFiasPath());
//$this->test[$item['PARENTGUID']][] = $item['AOGUID'];
$this->putToFile2(
array(
'ID' => $item['AOGUID'],
'ACTSTATUS' => $item['ACTSTATUS'],
'LIVESTATUS' => $item['LIVESTATUS'],
'NAME' => $item['FORMALNAME'],
'TYPE' => $item['SHORTNAME'],
'POSTALCODE' => $item['POSTALCODE']
),
'fias_tree',
strlen($item['PARENTGUID']) ? $item['PARENTGUID'] : 'root',
true
);
}
public function fiasGotOneAdd2DB($data)
{
$item = $data['__ATTR'];
$code = implode(' ', array(
$item['REGIONCODE'],
$item['AUTOCODE'],
$item['AREACODE'],
$item['CITYCODE'],
$item['CTARCODE'],
$item['PLACECODE'],
$item['STREETCODE'],
$item['EXTRCODE'],
$item['SEXTCODE'],
));
// if db works in cp1251
/*
$formalName = \CharsetConverter::ConvertCharset($item['FORMALNAME'], 'UTF-8', SITE_CHARSET);
$nameLC = \CharsetConverter::ConvertCharset($this->makeNameIndexKey($item['FORMALNAME']), 'UTF-8', SITE_CHARSET);
$shortName = \CharsetConverter::ConvertCharset($item['SHORTNAME'], 'UTF-8', SITE_CHARSET);
*/
$formalName = $item['FORMALNAME'];
$nameLC = $this->makeNameIndexKey($item['FORMALNAME']);
$shortName = $item['SHORTNAME'];
$this->fiasDB->insert(array(
'AOGUID' => $item['AOGUID'],
'PARENTGUID' => $item['PARENTGUID'],
'AOID' => $item['AOID'],
'NEXTID' => $item['NEXTID'],
'FORMALNAME' => $formalName,
'SHORTNAME' => $shortName,
'POSTALCODE' => $item['POSTALCODE'],
'ACTSTATUS' => $item['ACTSTATUS'],
'LIVESTATUS' => $item['LIVESTATUS'],
'NAME_LC' => $nameLC,
'CODE' => $code
));
}
private function getByAOID($aoid)
{
$id = explode('-', $aoid);
}
private function printCurrentFiasPath()
{
$path = array();
foreach($this->fiasCPath as $item)
$path[] = $item['NAME'];
return implode('>', $path);
}
private function manageFiasPath($item)
{
$guid = $item['AOGUID'];
$parentGUID = $item['PARENTGUID'];
//$this->truncateCurrentFiasPath($parentGUID);
//$this->fiasCPath[] = array('GUID' => $guid, 'NAME' => $item['FORMALNAME']);
}
private function truncateCurrentFiasPath($guid)
{
foreach($this->fiasCPath as $i => $node)
{
if($node['GUID'] == $guid)
{
array_splice($this->fiasCPath, $i + 1);
return;
}
}
}
#######################################################
### ABOUT FIAS PROCESS ROOT v2
#######################################################
public function mapFiasRootV2()
{
$this->cleanUpFile('fias_yamarket_links', 'rootv2');
$this->walkFias('fiasGotOneMapRootV2');
foreach($this->data['MAPS']['REGIONS'] as $id => $reg)
{
$foundId = '';
$foundName = '';
$additResults = '';
if(count($reg['MATCH']))
{
$foundId = $reg['MATCH'][0]['ID'];
$foundName = $reg['MATCH'][0]['NAME'];
array_shift($reg['MATCH']);
if(count($reg['MATCH']))
{
$additResults = array();
foreach($reg['MATCH'] as $additRes)
$additResults[] = $additRes['ID'].':"'.$additRes['NAME'].'"';
$additResults = implode(', ', $additResults);
}
}
$this->putToFile(
array(
'YAMARKET' => $id,
'YAMARKET_NAME' => $reg['NAME']['RU']['NAME'],
'AOGUID' => $foundId,
'FIAS_NAME' => $foundName,
'ADDITIONAL' => $additResults
),
'fias_yamarket_links',
'rootv2'
);
}
}
public function fiasGotOneMapRootV2($data)
{
$item = $data['__ATTR'];
$type = $item['SHORTNAME'];
$name = $this->makeNameIndexKey($item['FORMALNAME']);
if($this->sysMaps['FIAS2BASETYPE'][$type] == 'REGION') // this is fias-region
{
$result = array();
foreach($this->data['MAPS']['REGIONS'] as $id => &$node)
{
$rName = $this->makeNameIndexKey($node['NAME']['RU']['NAME']);
if($rName == $name || strpos($rName, $name) !== false)
{
$node['MATCH'][] = array(
'ID' => $item['AOGUID'],
'NAME' => $item['FORMALNAME'].' '.$item['SHORTNAME']
);
}
}
}
}
private function readFiasRootMapV2()
{
try
{
$csv = new Import\CSVReader('R', false);
$result = array();
$data = $csv->ReadBlock($this->getPoolFileName('fias_yamarket_links', 'rootv2', false));
foreach($data as $region)
$result[$region['YAMARKET']] = $region;
return $result;
}
catch(\Exception $e)
{
return array();
}
}
private function walkFias($callback, $limit = -1)
{
$sax = new SAXParser(array(
'watch4Tag' => 'Object',
'onEachParseResult' => array($this, $callback),
'limit' => $limit,
'collapseAttr' => true
));
$fd = fopen($_SERVER['DOCUMENT_ROOT'].$this->options['fiasAddrobjFile'], 'r');
while($block = fread($fd, 1024))
{
if(!$sax->putToParser($block))
break;
}
unset($sax);
}
#######################################################
### ABOUT MAIN DATA
#######################################################
private function buildRussiaPathIndex($bundle, $parentPath = array())
{
foreach($bundle as $id)
{
$node = $this->data['TREES']['MAIN']['NODES'][$id];
$name = $this->makeNameIndexKey($node['NAME']['RU']['NAME']);
//regions:
if($node['TYPE_CODE'] == 'REGION')
$this->data['MAPS']['REGIONS'][$id] = $node;
if(isset($this->data['TREES']['MAIN']['EDGES'][$id]))
$this->buildRussiaPathIndex($this->data['TREES']['MAIN']['EDGES'][$id], $ppp);
}
}
private function getMainTreeNodesOfType($bundle, $types = array(), &$buffer)
{
foreach($bundle as $id)
{
$node = $this->data['TREES']['MAIN']['NODES'][$id];
if(in_array($node['TYPE_CODE'], $types))
{
//$node['NAME_I'] = $this->makeNameIndexKey($node['NAME']['RU']['NAME']);
$buffer[$id] = $node;
}
if(isset($this->data['TREES']['MAIN']['EDGES'][$id]))
$this->getMainTreeNodesOfType($this->data['TREES']['MAIN']['EDGES'][$id], $types, $buffer);
}
}
#######################################################
### ABOUT FILE POOL
#######################################################
private function putToFile($data, $poolName, $fileSubname)
{
$dir = $_SERVER['DOCUMENT_ROOT'].$this->workDir.$this->filePools[$poolName]['DIR'];
if(!file_exists($dir))
mkdir($dir, 0755, true);
if(!isset($this->filePoolsp[$poolName][$fileSubname]))
{
$fd = $this->filePoolsp[$poolName][$fileSubname] = fopen($dir.$fileSubname.'.csv', 'w');
$head = implode(';', array_keys($data));
fputs($fd, $head.PHP_EOL);
$this->filePoolsp[$poolName][$fileSubname] = $fd;
}
fputs($this->filePoolsp[$poolName][$fileSubname], implode(';', $data).PHP_EOL);
}
private function putToFile2($data, $poolName, $fileSubname, $checkDir = false)
{
$dir = $_SERVER['DOCUMENT_ROOT'].$this->workDir.$this->filePools[$poolName]['DIR'];
$fName = $dir.$fileSubname.'.csv';
if($checkDir && !file_exists($dir))
mkdir($dir, 0755, true);
if(!file_exists($fName))
file_put_contents($fName, implode(';', array_keys($data)).PHP_EOL, FILE_APPEND);
file_put_contents($fName, implode(';', $data).PHP_EOL, FILE_APPEND);
}
private function cleanUpFile($poolName, $fileSubname)
{
$name = $_SERVER['DOCUMENT_ROOT'].$this->workDir.$this->filePools[$poolName]['DIR'].$fileSubname.'.csv';
if(file_exists($name))
unlink($name);
}
private function getPoolFileName($poolName, $fileSubname, $docRoot = true)
{
return ($docRoot ? $_SERVER['DOCUMENT_ROOT'] : '/').$this->workDir.$this->filePools[$poolName]['DIR'].$fileSubname.'.csv';
}
private function getPoolDirName($poolName, $docRoot = true)
{
return ($docRoot ? $_SERVER['DOCUMENT_ROOT'] : '/').$this->workDir.$this->filePools[$poolName]['DIR'];
}
private function cleanPoolDir($poolName)
{
$dir = $_SERVER['DOCUMENT_ROOT'].$this->workDir.$this->filePools[$poolName]['DIR'];
if(file_exists($dir))
system('rm -rf '.$dir);
mkdir($dir, 0755, true);
}
private function getDataFromCSV($poolName, $fileSubname)
{
try
{
$csv = new Import\CSVReader('R', false);
return $csv->ReadBlock($this->getPoolFileName($poolName, $fileSubname, false));
}
catch(\Exception $e)
{
return array();
}
}
private function putDataToCSV($data, $poolName, $fileSubname)
{
$fName = $this->getPoolFileName($poolName, $fileSubname);
if(file_exists($fName))
{
$header = implode(';', array_keys($data[0])).PHP_EOL;
file_put_contents($fName, $header);
foreach($data as $line)
file_put_contents($fName, implode(';', $line).PHP_EOL, FILE_APPEND);
}
}
#######################################################
### ABOUT COMPILER
#######################################################
private function mapFiasTypeToMain($fiasType)
{
return isset($this->sysMaps['FIAS2BASETYPE'][$fiasType]) ? $this->sysMaps['FIAS2BASETYPE'][$fiasType] : false;
}
private function makeNameIndexKey($name)
{
return trim(mb_strtolower($name, 'UTF-8'));
}
private function makeTypeGroupFile($file = '')
{
$fd = $this->fileOpen(strlen($file) ? $file : self::GROUP_FILE);
fputs($fd, implode(';', $this->headers['GROUP_FILE']).PHP_EOL);
foreach($this->typeGroups as $code => $group)
{
$line = array();
foreach($this->headers['GROUP_FILE'] as $colCode)
$line[] = is_array($group[$colCode]) ? implode(':', $group[$colCode]) : $group[$colCode];
fputs($fd, implode(';', $line).PHP_EOL);
}
fclose($fd);
}
private function makeNext()
{
$next = $this->queueShift();
$bundle = $this->getBundleFromFile($next);
if(!empty($bundle))
{
foreach($bundle as $item)
{
$this->putToGroups($item);
if($item['CHILDREN_COUNT'] > 0)
$this->queue[] = $item['ID'];
}
}
return empty($this->queue);
}
private function queueShift()
{
if($this->queue !== false)
return array_shift($this->queue);
return 'root';
}
private function getBundleFromFile($id)
{
$data = unserialize(file_get_contents($_SERVER['DOCUMENT_ROOT'].$this->grabbedStuffDir.$id));
foreach($data as $k => &$item)
{
if(in_array($item['NAME'], array('Прочее', 'Общероссийские', 'Универсальное', 'Другие города региона')))
{
unset($data[$k]);
continue;
}
//$item['NAME'] = \CharsetConverter::ConvertCharset($item['NAME'], 'UTF-8', SITE_CHARSET); // temp
if(isset($this->typeMap[$item['TYPE_CODE']]))
$item['TYPE_CODE'] = $this->typeMap[$item['TYPE_CODE']];
// type "OTHER" which is a child of type "REGION" is actually a "SUBREGION"
if($item['TYPE_CODE'] == 'OTHER')
{
$parentType = $this->yaIdType[$item['PARENT_ID']];
if($parentType == 'REGION')
$item['TYPE_CODE'] = 'SUBREGION';
}
$code = $this->addLeadingZero($this->codeOffset, $this->leading);
$this->yaIdType[$item['ID']] = $item['TYPE_CODE'];
$this->relations[$item['CODE']] = $item['PARENT_CODE'];
$this->code2type[$item['CODE']] = $item['TYPE_CODE'];
$ruName = $item['NAME'];
//if($this->optionConvertNames)
// $ruName = \CharsetConverter::ConvertCharset($ruName, 'UTF-8', SITE_CHARSET);
$item['NAME'] = array();
$item['NAME']['RU']['NAME'] = $ruName.($this->options['includeYaInfo2Name'] ? ' ('.$item['TYPE_CODE'].', '.$item['ID'].')' : '');
//$item['NAME.EN.NAME'] = '[no-translation]'; // attach translations from old import files
//$item['NAME.UA.NAME'] = '[no-translation]'; // attach translations from old import files
$item['EXT']['YAMARKET'][] = $item['ID'];
//unset($item['ID']);
//unset($item['PARENT_ID']);
}
return $data;
}
private function getParentOfType($code, $types)
{
if(empty($types))
return '';
$nextCode = $code;
$i = -1;
while($nextCode)
{
$i++;
if($i > 50)
throw new Main\SystemException('Recursion gone too deep when trying to find parent of type');
if(isset($types[$this->code2type[$nextCode]]))
return $nextCode;
$nextCode = $this->relations[$nextCode];
}
return '';
}
private function putToGroups($item)
{
foreach($this->typeGroups as $gCode => &$group)
{
//_dump_r('Item '.$item['NAME.RU.NAME'].' ('.$item['TYPE_CODE'].')');
if(!isset($group['I_TYPES'][$item['TYPE_CODE']]))
continue;
//_dump_r('Goes to group: '.$gCode);
$baseParent = $this->getParentOfType($item['CODE'], $this->typeGroups[$group['PARENT']]['I_TYPES']);
//_dump_r('Base parent is: '.$baseParent);
if(!$group['FD'][$baseParent])
{
$fName = str_replace(array(
'%BASE_PARENT_ITEM_CODE%',
'%CODE%'
), array(
$baseParent,
ToLower($gCode)
), $group['FILE_NAME_TEMPLATE']);
$group['FD'][$baseParent] = $this->fileOpen($fName);
fputs($group['FD'][$baseParent], implode(';', $this->headers[$group['HEADER']]).PHP_EOL);
}
$header = $this->headers[$group['HEADER']];
$line = array();
foreach($header as $code)
$line[] = isset($item[$code]) ? $item[$code] : '';
fputs($group['FD'][$baseParent], implode(';', $line).PHP_EOL);
}
}
private function fileOpen($name)
{
return fopen($_SERVER['DOCUMENT_ROOT'].$this->workDir.self::OUTPUT_DIR.$name, 'w');
}
private static function addLeadingZero($value, $length)
{
if(strlen($value) >= $length)
return $value;
$diff = abs($length - strlen($value));
for($i = 0; $i < $diff; $i++)
$value = '0'.$value;
return $value;
}
#######################################################
### ABOUT DATA
#######################################################
private function storeTemporalData($dataCode, $data)
{
$dir = $_SERVER['DOCUMENT_ROOT'].$this->workDir.self::TMP_DATA_DIR;
if(!file_exists($dir))
mkdir($dir, 0755, true);
file_put_contents($dir.$dataCode, serialize($data));
}
private function getStoredTemporalData($dataCode)
{
$file = $_SERVER['DOCUMENT_ROOT'].$this->workDir.self::TMP_DATA_DIR.$dataCode;
if(is_readable($file))
return unserialize(file_get_contents($file));
else
return array();
}
private function cleanTemporalData($dataCode)
{
$file = $_SERVER['DOCUMENT_ROOT'].$this->workDir.self::TMP_DATA_DIR.$dataCode;
if(is_readable($file))
unlink($file);
}
/*
private function dropTemporalFile()
{
$file = $_SERVER['DOCUMENT_ROOT'].$this->workDir.self::TMP_DATA_FILE;
if(is_readable($file))
unlink($file);
}
*/
public function cleanOutput()
{
$file = $_SERVER['DOCUMENT_ROOT'].$this->workDir.self::OUTPUT_FILE;
if(is_readable($file))
unlink($file);
}
public function output($data, $important = true)
{
if(!$important)
return false;
ob_start();
print_r($data);
$data = ob_get_contents();
ob_end_clean();
file_put_contents($_SERVER['DOCUMENT_ROOT'].$this->workDir.self::OUTPUT_FILE, $data.PHP_EOL, FILE_APPEND);
}
}
Zerion Mini Shell 1.0